2024-11-09 06:15:21,764 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 06:15:21,780 main DEBUG Took 0.014240 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 06:15:21,781 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 06:15:21,781 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 06:15:21,783 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 06:15:21,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,794 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 06:15:21,809 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,811 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,812 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,812 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,813 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,814 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,814 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,815 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,816 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,817 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,817 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,818 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,818 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,819 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,819 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,820 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,820 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,821 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,822 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,822 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 06:15:21,823 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,824 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 06:15:21,826 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 06:15:21,828 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 06:15:21,830 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 06:15:21,831 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 06:15:21,833 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 06:15:21,834 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 06:15:21,845 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 06:15:21,848 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 06:15:21,850 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 06:15:21,851 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 06:15:21,851 main DEBUG createAppenders(={Console}) 2024-11-09 06:15:21,852 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-09 06:15:21,853 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 06:15:21,853 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-09 06:15:21,854 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 06:15:21,855 main DEBUG OutputStream closed 2024-11-09 06:15:21,855 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 06:15:21,856 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 06:15:21,856 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-09 06:15:21,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 06:15:21,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 06:15:21,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 06:15:21,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 06:15:21,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 06:15:21,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 06:15:21,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 06:15:21,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 06:15:21,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 06:15:21,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 06:15:21,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 06:15:21,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 06:15:21,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 06:15:21,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 06:15:21,964 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 06:15:21,964 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 06:15:21,965 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 06:15:21,966 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 06:15:21,968 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 06:15:21,969 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-09 06:15:21,969 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 06:15:21,970 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-09T06:15:22,246 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8 2024-11-09 06:15:22,250 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 06:15:22,251 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T06:15:22,265 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-09T06:15:22,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T06:15:22,324 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/cluster_d5b7a4cc-a1e2-6520-f0a5-179e578abc6c, deleteOnExit=true 2024-11-09T06:15:22,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-09T06:15:22,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/test.cache.data in system properties and HBase conf 2024-11-09T06:15:22,328 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T06:15:22,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/hadoop.log.dir in system properties and HBase conf 2024-11-09T06:15:22,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T06:15:22,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T06:15:22,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-09T06:15:22,476 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T06:15:22,626 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T06:15:22,632 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T06:15:22,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T06:15:22,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T06:15:22,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T06:15:22,635 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T06:15:22,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T06:15:22,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T06:15:22,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T06:15:22,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T06:15:22,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/nfs.dump.dir in system properties and HBase conf 2024-11-09T06:15:22,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/java.io.tmpdir in system properties and HBase conf 2024-11-09T06:15:22,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T06:15:22,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T06:15:22,640 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T06:15:24,057 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T06:15:24,187 INFO [Time-limited test {}] log.Log(170): Logging initialized @3223ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T06:15:24,271 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T06:15:24,351 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T06:15:24,377 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T06:15:24,378 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T06:15:24,380 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T06:15:24,396 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T06:15:24,401 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/hadoop.log.dir/,AVAILABLE} 2024-11-09T06:15:24,402 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T06:15:24,657 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/java.io.tmpdir/jetty-localhost-40277-hadoop-hdfs-3_4_1-tests_jar-_-any-12356402890368311861/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T06:15:24,668 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40277} 2024-11-09T06:15:24,669 INFO [Time-limited test {}] server.Server(415): Started @3706ms 2024-11-09T06:15:25,330 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T06:15:25,346 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T06:15:25,348 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T06:15:25,348 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T06:15:25,349 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T06:15:25,350 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/hadoop.log.dir/,AVAILABLE} 2024-11-09T06:15:25,351 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T06:15:25,455 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/java.io.tmpdir/jetty-localhost-40537-hadoop-hdfs-3_4_1-tests_jar-_-any-2626076661542633615/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T06:15:25,456 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:40537} 2024-11-09T06:15:25,456 INFO [Time-limited test {}] server.Server(415): Started @4493ms 2024-11-09T06:15:25,518 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T06:15:27,060 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/cluster_d5b7a4cc-a1e2-6520-f0a5-179e578abc6c/data/data2/current/BP-1465325163-172.17.0.3-1731132923455/current, will proceed with Du for space computation calculation, 2024-11-09T06:15:27,060 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/cluster_d5b7a4cc-a1e2-6520-f0a5-179e578abc6c/data/data1/current/BP-1465325163-172.17.0.3-1731132923455/current, will proceed with Du for space computation calculation, 2024-11-09T06:15:27,131 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T06:15:27,200 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x959aacaf621d7712 with lease ID 0xb34b5567072d459a: Processing first storage report for DS-ca5e037a-b46b-4653-aee7-c9b65584e3ef from datanode DatanodeRegistration(127.0.0.1:41571, datanodeUuid=fa7624cf-f0f4-47e0-ac1f-fe9b1ca50877, infoPort=38639, infoSecurePort=0, ipcPort=37629, storageInfo=lv=-57;cid=testClusterID;nsid=608021774;c=1731132923455) 2024-11-09T06:15:27,202 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x959aacaf621d7712 with lease ID 0xb34b5567072d459a: from storage DS-ca5e037a-b46b-4653-aee7-c9b65584e3ef node DatanodeRegistration(127.0.0.1:41571, datanodeUuid=fa7624cf-f0f4-47e0-ac1f-fe9b1ca50877, infoPort=38639, infoSecurePort=0, ipcPort=37629, storageInfo=lv=-57;cid=testClusterID;nsid=608021774;c=1731132923455), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-09T06:15:27,203 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x959aacaf621d7712 with lease ID 0xb34b5567072d459a: Processing first storage report for DS-8e5a7176-fd51-4e8a-b49d-996814f95792 from datanode DatanodeRegistration(127.0.0.1:41571, datanodeUuid=fa7624cf-f0f4-47e0-ac1f-fe9b1ca50877, infoPort=38639, infoSecurePort=0, ipcPort=37629, storageInfo=lv=-57;cid=testClusterID;nsid=608021774;c=1731132923455) 2024-11-09T06:15:27,204 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x959aacaf621d7712 with lease ID 0xb34b5567072d459a: from storage DS-8e5a7176-fd51-4e8a-b49d-996814f95792 node DatanodeRegistration(127.0.0.1:41571, datanodeUuid=fa7624cf-f0f4-47e0-ac1f-fe9b1ca50877, infoPort=38639, infoSecurePort=0, ipcPort=37629, storageInfo=lv=-57;cid=testClusterID;nsid=608021774;c=1731132923455), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T06:15:27,259 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8 2024-11-09T06:15:27,406 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/cluster_d5b7a4cc-a1e2-6520-f0a5-179e578abc6c/zookeeper_0, clientPort=62248, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/cluster_d5b7a4cc-a1e2-6520-f0a5-179e578abc6c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/cluster_d5b7a4cc-a1e2-6520-f0a5-179e578abc6c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T06:15:27,418 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62248 2024-11-09T06:15:27,434 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T06:15:27,439 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T06:15:27,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741825_1001 (size=7) 2024-11-09T06:15:28,101 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f with version=8 2024-11-09T06:15:28,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/hbase-staging 2024-11-09T06:15:28,215 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T06:15:28,483 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4a84bcce7bad:0 server-side Connection retries=45 2024-11-09T06:15:28,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T06:15:28,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T06:15:28,498 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T06:15:28,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T06:15:28,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T06:15:28,662 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T06:15:28,721 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T06:15:28,734 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T06:15:28,739 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T06:15:28,775 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8506 (auto-detected) 2024-11-09T06:15:28,776 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-09T06:15:28,797 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37069 2024-11-09T06:15:28,819 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37069 connecting to ZooKeeper ensemble=127.0.0.1:62248 2024-11-09T06:15:28,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:370690x0, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T06:15:28,950 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37069-0x1011e3c29400000 connected 2024-11-09T06:15:29,038 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T06:15:29,041 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T06:15:29,052 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T06:15:29,056 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f, hbase.cluster.distributed=false 2024-11-09T06:15:29,081 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T06:15:29,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37069 2024-11-09T06:15:29,087 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37069 2024-11-09T06:15:29,087 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37069 2024-11-09T06:15:29,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37069 2024-11-09T06:15:29,092 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37069 2024-11-09T06:15:29,210 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4a84bcce7bad:0 server-side Connection retries=45 2024-11-09T06:15:29,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T06:15:29,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T06:15:29,212 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T06:15:29,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T06:15:29,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T06:15:29,215 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T06:15:29,217 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T06:15:29,218 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46317 2024-11-09T06:15:29,220 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46317 connecting to ZooKeeper ensemble=127.0.0.1:62248 2024-11-09T06:15:29,222 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T06:15:29,224 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T06:15:29,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:463170x0, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T06:15:29,244 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46317-0x1011e3c29400001 connected 2024-11-09T06:15:29,244 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:463170x0, quorum=127.0.0.1:62248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T06:15:29,249 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T06:15:29,260 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T06:15:29,263 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T06:15:29,270 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T06:15:29,270 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46317 2024-11-09T06:15:29,271 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46317 2024-11-09T06:15:29,272 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46317 2024-11-09T06:15:29,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46317 2024-11-09T06:15:29,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46317 2024-11-09T06:15:29,293 DEBUG [M:0;4a84bcce7bad:37069 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4a84bcce7bad:37069 2024-11-09T06:15:29,294 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4a84bcce7bad,37069,1731132928314 2024-11-09T06:15:29,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T06:15:29,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T06:15:29,313 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4a84bcce7bad,37069,1731132928314 2024-11-09T06:15:29,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:29,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T06:15:29,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:29,355 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T06:15:29,356 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4a84bcce7bad,37069,1731132928314 from backup master directory 2024-11-09T06:15:29,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4a84bcce7bad,37069,1731132928314 2024-11-09T06:15:29,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T06:15:29,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T06:15:29,370 WARN [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T06:15:29,371 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4a84bcce7bad,37069,1731132928314 2024-11-09T06:15:29,373 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T06:15:29,374 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T06:15:29,430 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/hbase.id] with ID: 651e661e-20b2-435b-964a-2cfde8619a3d 2024-11-09T06:15:29,430 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/.tmp/hbase.id 2024-11-09T06:15:29,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741826_1002 (size=42) 2024-11-09T06:15:29,843 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/.tmp/hbase.id]:[hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/hbase.id] 2024-11-09T06:15:29,903 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T06:15:29,908 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T06:15:29,930 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-09T06:15:29,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:29,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:29,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741827_1003 (size=196) 2024-11-09T06:15:30,374 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T06:15:30,376 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T06:15:30,392 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:30,396 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T06:15:30,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741828_1004 (size=1189) 2024-11-09T06:15:30,844 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store 2024-11-09T06:15:30,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741829_1005 (size=34) 2024-11-09T06:15:31,267 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T06:15:31,271 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:15:31,272 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T06:15:31,273 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T06:15:31,273 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T06:15:31,275 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T06:15:31,275 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T06:15:31,276 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T06:15:31,277 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731132931272Disabling compacts and flushes for region at 1731132931272Disabling writes for close at 1731132931275 (+3 ms)Writing region close event to WAL at 1731132931275Closed at 1731132931276 (+1 ms) 2024-11-09T06:15:31,280 WARN [master/4a84bcce7bad:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/.initializing 2024-11-09T06:15:31,280 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/WALs/4a84bcce7bad,37069,1731132928314 2024-11-09T06:15:31,288 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T06:15:31,306 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4a84bcce7bad%2C37069%2C1731132928314, suffix=, logDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/WALs/4a84bcce7bad,37069,1731132928314, archiveDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/oldWALs, maxLogs=10 2024-11-09T06:15:31,331 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/WALs/4a84bcce7bad,37069,1731132928314/4a84bcce7bad%2C37069%2C1731132928314.1731132931310, exclude list is [], retry=0 2024-11-09T06:15:31,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41571,DS-ca5e037a-b46b-4653-aee7-c9b65584e3ef,DISK] 2024-11-09T06:15:31,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T06:15:31,391 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/WALs/4a84bcce7bad,37069,1731132928314/4a84bcce7bad%2C37069%2C1731132928314.1731132931310 2024-11-09T06:15:31,392 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38639:38639)] 2024-11-09T06:15:31,393 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T06:15:31,393 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:15:31,396 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,397 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,464 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T06:15:31,468 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:31,471 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T06:15:31,471 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T06:15:31,476 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:31,477 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:15:31,477 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T06:15:31,480 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:31,481 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:15:31,481 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T06:15:31,485 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:31,486 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:15:31,487 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,491 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,492 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,498 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,498 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,501 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T06:15:31,505 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T06:15:31,511 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T06:15:31,512 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65465575, jitterRate=-0.02448691427707672}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T06:15:31,518 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731132931408Initializing all the Stores at 1731132931411 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731132931412 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132931413 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132931413Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132931413Cleaning up temporary data from old regions at 1731132931498 (+85 ms)Region opened successfully at 1731132931518 (+20 ms) 2024-11-09T06:15:31,519 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T06:15:31,557 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5878f5d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4a84bcce7bad/172.17.0.3:0 2024-11-09T06:15:31,597 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T06:15:31,612 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T06:15:31,613 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T06:15:31,617 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T06:15:31,618 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T06:15:31,626 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-09T06:15:31,626 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T06:15:31,662 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T06:15:31,678 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T06:15:31,689 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T06:15:31,691 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T06:15:31,693 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T06:15:31,710 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T06:15:31,713 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T06:15:31,718 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T06:15:31,731 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T06:15:31,734 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T06:15:31,741 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T06:15:31,758 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T06:15:31,769 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T06:15:31,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T06:15:31,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T06:15:31,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:31,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:31,787 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4a84bcce7bad,37069,1731132928314, sessionid=0x1011e3c29400000, setting cluster-up flag (Was=false) 2024-11-09T06:15:31,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:31,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:32,026 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T06:15:32,029 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4a84bcce7bad,37069,1731132928314 2024-11-09T06:15:32,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:32,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:32,145 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T06:15:32,147 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4a84bcce7bad,37069,1731132928314 2024-11-09T06:15:32,153 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T06:15:32,186 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(746): ClusterId : 651e661e-20b2-435b-964a-2cfde8619a3d 2024-11-09T06:15:32,188 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T06:15:32,217 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T06:15:32,217 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T06:15:32,227 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T06:15:32,228 DEBUG [RS:0;4a84bcce7bad:46317 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@719e106d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4a84bcce7bad/172.17.0.3:0 2024-11-09T06:15:32,231 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T06:15:32,239 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T06:15:32,242 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4a84bcce7bad:46317 2024-11-09T06:15:32,245 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T06:15:32,245 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T06:15:32,246 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T06:15:32,246 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T06:15:32,248 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(2659): reportForDuty to master=4a84bcce7bad,37069,1731132928314 with port=46317, startcode=1731132929173 2024-11-09T06:15:32,250 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4a84bcce7bad,37069,1731132928314 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T06:15:32,257 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4a84bcce7bad:0, corePoolSize=5, maxPoolSize=5 2024-11-09T06:15:32,257 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4a84bcce7bad:0, corePoolSize=5, maxPoolSize=5 2024-11-09T06:15:32,257 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4a84bcce7bad:0, corePoolSize=5, maxPoolSize=5 2024-11-09T06:15:32,257 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4a84bcce7bad:0, corePoolSize=5, maxPoolSize=5 2024-11-09T06:15:32,257 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4a84bcce7bad:0, corePoolSize=10, maxPoolSize=10 2024-11-09T06:15:32,258 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,258 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4a84bcce7bad:0, corePoolSize=2, maxPoolSize=2 2024-11-09T06:15:32,258 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,260 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731132962260 2024-11-09T06:15:32,261 DEBUG [RS:0;4a84bcce7bad:46317 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T06:15:32,263 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T06:15:32,264 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T06:15:32,264 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T06:15:32,265 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T06:15:32,268 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T06:15:32,268 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T06:15:32,269 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T06:15:32,269 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T06:15:32,272 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:32,272 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T06:15:32,273 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,287 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T06:15:32,288 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T06:15:32,289 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T06:15:32,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741831_1007 (size=1321) 2024-11-09T06:15:32,296 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T06:15:32,297 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T06:15:32,301 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4a84bcce7bad:0:becomeActiveMaster-HFileCleaner.large.0-1731132932298,5,FailOnTimeoutGroup] 2024-11-09T06:15:32,302 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4a84bcce7bad:0:becomeActiveMaster-HFileCleaner.small.0-1731132932301,5,FailOnTimeoutGroup] 2024-11-09T06:15:32,303 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,304 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T06:15:32,306 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,308 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,309 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36339, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T06:15:32,324 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37069 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:32,327 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37069 {}] master.ServerManager(517): Registering regionserver=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:32,347 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f 2024-11-09T06:15:32,347 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43365 2024-11-09T06:15:32,348 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T06:15:32,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T06:15:32,364 DEBUG [RS:0;4a84bcce7bad:46317 {}] zookeeper.ZKUtil(111): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:32,364 WARN [RS:0;4a84bcce7bad:46317 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T06:15:32,365 INFO [RS:0;4a84bcce7bad:46317 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T06:15:32,365 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/WALs/4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:32,367 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4a84bcce7bad,46317,1731132929173] 2024-11-09T06:15:32,398 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T06:15:32,422 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T06:15:32,429 INFO [RS:0;4a84bcce7bad:46317 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T06:15:32,430 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,431 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T06:15:32,439 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T06:15:32,441 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,442 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,442 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,442 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,442 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,443 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,443 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4a84bcce7bad:0, corePoolSize=2, maxPoolSize=2 2024-11-09T06:15:32,443 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,443 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,444 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,444 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,444 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,444 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4a84bcce7bad:0, corePoolSize=1, maxPoolSize=1 2024-11-09T06:15:32,444 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4a84bcce7bad:0, corePoolSize=3, maxPoolSize=3 2024-11-09T06:15:32,445 DEBUG [RS:0;4a84bcce7bad:46317 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0, corePoolSize=3, maxPoolSize=3 2024-11-09T06:15:32,446 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,446 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,446 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,446 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,447 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,447 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(168): Chore ScheduledChore name=4a84bcce7bad,46317,1731132929173-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T06:15:32,475 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T06:15:32,479 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(168): Chore ScheduledChore name=4a84bcce7bad,46317,1731132929173-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,479 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,480 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.Replication(171): 4a84bcce7bad,46317,1731132929173 started 2024-11-09T06:15:32,512 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:32,513 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1482): Serving as 4a84bcce7bad,46317,1731132929173, RpcServer on 4a84bcce7bad/172.17.0.3:46317, sessionid=0x1011e3c29400001 2024-11-09T06:15:32,514 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T06:15:32,514 DEBUG [RS:0;4a84bcce7bad:46317 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:32,515 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4a84bcce7bad,46317,1731132929173' 2024-11-09T06:15:32,515 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T06:15:32,516 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T06:15:32,517 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T06:15:32,517 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T06:15:32,518 DEBUG [RS:0;4a84bcce7bad:46317 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:32,518 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4a84bcce7bad,46317,1731132929173' 2024-11-09T06:15:32,518 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T06:15:32,519 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T06:15:32,520 DEBUG [RS:0;4a84bcce7bad:46317 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T06:15:32,520 INFO [RS:0;4a84bcce7bad:46317 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T06:15:32,520 INFO [RS:0;4a84bcce7bad:46317 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T06:15:32,628 INFO [RS:0;4a84bcce7bad:46317 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T06:15:32,637 INFO [RS:0;4a84bcce7bad:46317 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4a84bcce7bad%2C46317%2C1731132929173, suffix=, logDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/WALs/4a84bcce7bad,46317,1731132929173, archiveDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/oldWALs, maxLogs=32 2024-11-09T06:15:32,660 DEBUG [RS:0;4a84bcce7bad:46317 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/WALs/4a84bcce7bad,46317,1731132929173/4a84bcce7bad%2C46317%2C1731132929173.1731132932641, exclude list is [], retry=0 2024-11-09T06:15:32,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41571,DS-ca5e037a-b46b-4653-aee7-c9b65584e3ef,DISK] 2024-11-09T06:15:32,671 INFO [RS:0;4a84bcce7bad:46317 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/WALs/4a84bcce7bad,46317,1731132929173/4a84bcce7bad%2C46317%2C1731132929173.1731132932641 2024-11-09T06:15:32,671 DEBUG [RS:0;4a84bcce7bad:46317 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38639:38639)] 2024-11-09T06:15:32,700 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T06:15:32,700 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f 2024-11-09T06:15:32,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741833_1009 (size=32) 2024-11-09T06:15:33,115 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:15:33,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T06:15:33,133 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T06:15:33,133 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:33,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T06:15:33,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T06:15:33,139 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T06:15:33,139 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:33,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T06:15:33,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T06:15:33,145 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T06:15:33,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:33,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T06:15:33,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T06:15:33,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T06:15:33,151 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:33,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T06:15:33,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T06:15:33,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740 2024-11-09T06:15:33,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740 2024-11-09T06:15:33,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T06:15:33,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T06:15:33,163 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T06:15:33,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T06:15:33,173 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T06:15:33,174 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67288116, jitterRate=0.00267106294631958}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T06:15:33,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731132933116Initializing all the Stores at 1731132933119 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731132933119Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731132933119Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132933119Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731132933119Cleaning up temporary data from old regions at 1731132933162 (+43 ms)Region opened successfully at 1731132933178 (+16 ms) 2024-11-09T06:15:33,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T06:15:33,179 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T06:15:33,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T06:15:33,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T06:15:33,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T06:15:33,183 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T06:15:33,183 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731132933179Disabling compacts and flushes for region at 1731132933179Disabling writes for close at 1731132933179Writing region close event to WAL at 1731132933182 (+3 ms)Closed at 1731132933183 (+1 ms) 2024-11-09T06:15:33,187 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T06:15:33,187 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T06:15:33,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T06:15:33,208 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T06:15:33,211 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T06:15:33,364 DEBUG [4a84bcce7bad:37069 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T06:15:33,378 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:33,399 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4a84bcce7bad,46317,1731132929173, state=OPENING 2024-11-09T06:15:33,420 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T06:15:33,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:33,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:15:33,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T06:15:33,434 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T06:15:33,436 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T06:15:33,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:15:33,624 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T06:15:33,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36831, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T06:15:33,641 INFO [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T06:15:33,642 INFO [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T06:15:33,643 INFO [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T06:15:33,647 INFO [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4a84bcce7bad%2C46317%2C1731132929173.meta, suffix=.meta, logDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/WALs/4a84bcce7bad,46317,1731132929173, archiveDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/oldWALs, maxLogs=32 2024-11-09T06:15:33,671 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/WALs/4a84bcce7bad,46317,1731132929173/4a84bcce7bad%2C46317%2C1731132929173.meta.1731132933650.meta, exclude list is [], retry=0 2024-11-09T06:15:33,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41571,DS-ca5e037a-b46b-4653-aee7-c9b65584e3ef,DISK] 2024-11-09T06:15:33,683 INFO [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/WALs/4a84bcce7bad,46317,1731132929173/4a84bcce7bad%2C46317%2C1731132929173.meta.1731132933650.meta 2024-11-09T06:15:33,684 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38639:38639)] 2024-11-09T06:15:33,684 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T06:15:33,687 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T06:15:33,691 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T06:15:33,698 INFO [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T06:15:33,705 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T06:15:33,706 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:15:33,707 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T06:15:33,707 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T06:15:33,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T06:15:33,713 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T06:15:33,713 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:33,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T06:15:33,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T06:15:33,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T06:15:33,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:33,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T06:15:33,719 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T06:15:33,721 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T06:15:33,721 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:33,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T06:15:33,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T06:15:33,726 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T06:15:33,726 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:33,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T06:15:33,728 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T06:15:33,730 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740 2024-11-09T06:15:33,734 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740 2024-11-09T06:15:33,738 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T06:15:33,739 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T06:15:33,740 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T06:15:33,745 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T06:15:33,748 INFO [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63056877, jitterRate=-0.06037931144237518}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T06:15:33,748 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T06:15:33,751 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731132933708Writing region info on filesystem at 1731132933708Initializing all the Stores at 1731132933710 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731132933710Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731132933710Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132933711 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731132933711Cleaning up temporary data from old regions at 1731132933739 (+28 ms)Running coprocessor post-open hooks at 1731132933748 (+9 ms)Region opened successfully at 1731132933751 (+3 ms) 2024-11-09T06:15:33,760 INFO [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731132933613 2024-11-09T06:15:33,778 DEBUG [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T06:15:33,779 INFO [RS_OPEN_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T06:15:33,781 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:33,787 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4a84bcce7bad,46317,1731132929173, state=OPEN 2024-11-09T06:15:33,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T06:15:33,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T06:15:33,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T06:15:33,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T06:15:33,867 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:33,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T06:15:33,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4a84bcce7bad,46317,1731132929173 in 429 msec 2024-11-09T06:15:33,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T06:15:33,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 682 msec 2024-11-09T06:15:33,888 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T06:15:33,888 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T06:15:33,917 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:33,919 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:33,943 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:33,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34045, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:33,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7770 sec 2024-11-09T06:15:33,972 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731132933972, completionTime=-1 2024-11-09T06:15:33,975 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T06:15:33,975 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T06:15:34,007 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-09T06:15:34,008 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731132994008 2024-11-09T06:15:34,008 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731133054008 2024-11-09T06:15:34,008 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 32 msec 2024-11-09T06:15:34,011 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a84bcce7bad,37069,1731132928314-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:34,012 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a84bcce7bad,37069,1731132928314-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:34,012 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a84bcce7bad,37069,1731132928314-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:34,014 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4a84bcce7bad:37069, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:34,015 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:34,016 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:34,023 DEBUG [master/4a84bcce7bad:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T06:15:34,050 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.679sec 2024-11-09T06:15:34,051 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T06:15:34,053 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T06:15:34,054 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T06:15:34,055 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T06:15:34,055 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T06:15:34,056 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a84bcce7bad,37069,1731132928314-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T06:15:34,057 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a84bcce7bad,37069,1731132928314-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T06:15:34,069 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T06:15:34,070 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T06:15:34,070 INFO [master/4a84bcce7bad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a84bcce7bad,37069,1731132928314-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T06:15:34,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:34,113 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T06:15:34,113 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T06:15:34,117 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:15:34,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:15:34,130 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:15:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:15:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:15:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775efbc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:15:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:15:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:34,143 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:15:34,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:34,147 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:34,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:34,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:34,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:34,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4a84bcce7bad,37069,1731132928314 2024-11-09T06:15:34,199 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=343, ProcessCount=11, AvailableMemoryMB=2429 2024-11-09T06:15:34,211 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T06:15:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a84bcce7bad,37069,1731132928314 2024-11-09T06:15:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c35b008 2024-11-09T06:15:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T06:15:34,243 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T06:15:34,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T06:15:34,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T06:15:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T06:15:34,277 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T06:15:34,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-09T06:15:34,280 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:34,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T06:15:34,286 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T06:15:34,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741835_1011 (size=1061) 2024-11-09T06:15:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T06:15:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T06:15:34,708 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f 2024-11-09T06:15:34,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741836_1012 (size=53) 2024-11-09T06:15:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T06:15:35,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:15:35,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4cec2ccd1a271d540c1f369d10f41e7f, disabling compactions & flushes 2024-11-09T06:15:35,121 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:35,122 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:35,122 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. after waiting 0 ms 2024-11-09T06:15:35,122 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:35,122 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:35,122 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4cec2ccd1a271d540c1f369d10f41e7f: Waiting for close lock at 1731132935121Disabling compacts and flushes for region at 1731132935121Disabling writes for close at 1731132935122 (+1 ms)Writing region close event to WAL at 1731132935122Closed at 1731132935122 2024-11-09T06:15:35,125 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T06:15:35,131 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731132935125"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731132935125"}]},"ts":"1731132935125"} 2024-11-09T06:15:35,136 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T06:15:35,139 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T06:15:35,141 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731132935139"}]},"ts":"1731132935139"} 2024-11-09T06:15:35,147 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T06:15:35,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4cec2ccd1a271d540c1f369d10f41e7f, ASSIGN}] 2024-11-09T06:15:35,151 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4cec2ccd1a271d540c1f369d10f41e7f, ASSIGN 2024-11-09T06:15:35,154 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4cec2ccd1a271d540c1f369d10f41e7f, ASSIGN; state=OFFLINE, location=4a84bcce7bad,46317,1731132929173; forceNewPlan=false, retain=false 2024-11-09T06:15:35,306 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4cec2ccd1a271d540c1f369d10f41e7f, regionState=OPENING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:35,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4cec2ccd1a271d540c1f369d10f41e7f, ASSIGN because future has completed 2024-11-09T06:15:35,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:15:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T06:15:35,485 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:35,486 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} 2024-11-09T06:15:35,486 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,486 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:15:35,487 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,487 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,490 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,494 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:15:35,494 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cec2ccd1a271d540c1f369d10f41e7f columnFamilyName A 2024-11-09T06:15:35,494 DEBUG [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:35,496 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] regionserver.HStore(327): Store=4cec2ccd1a271d540c1f369d10f41e7f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:15:35,496 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,498 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:15:35,499 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cec2ccd1a271d540c1f369d10f41e7f columnFamilyName B 2024-11-09T06:15:35,499 DEBUG [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:35,500 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] regionserver.HStore(327): Store=4cec2ccd1a271d540c1f369d10f41e7f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:15:35,500 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,503 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:15:35,503 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cec2ccd1a271d540c1f369d10f41e7f columnFamilyName C 2024-11-09T06:15:35,504 DEBUG [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:15:35,505 INFO [StoreOpener-4cec2ccd1a271d540c1f369d10f41e7f-1 {}] regionserver.HStore(327): Store=4cec2ccd1a271d540c1f369d10f41e7f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:15:35,505 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:35,505 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,508 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,509 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,511 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,512 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,513 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T06:15:35,516 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,521 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T06:15:35,522 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4cec2ccd1a271d540c1f369d10f41e7f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64004196, jitterRate=-0.046263158321380615}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T06:15:35,522 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:35,524 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4cec2ccd1a271d540c1f369d10f41e7f: Running coprocessor pre-open hook at 1731132935487Writing region info on filesystem at 1731132935487Initializing all the Stores at 1731132935489 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132935489Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132935489Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132935489Cleaning up temporary data from old regions at 1731132935512 (+23 ms)Running coprocessor post-open hooks at 1731132935522 (+10 ms)Region opened successfully at 1731132935523 (+1 ms) 2024-11-09T06:15:35,526 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., pid=6, masterSystemTime=1731132935477 2024-11-09T06:15:35,530 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:35,530 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:35,532 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4cec2ccd1a271d540c1f369d10f41e7f, regionState=OPEN, openSeqNum=2, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:35,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:15:35,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T06:15:35,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 in 222 msec 2024-11-09T06:15:35,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T06:15:35,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4cec2ccd1a271d540c1f369d10f41e7f, ASSIGN in 399 msec 2024-11-09T06:15:35,555 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T06:15:35,556 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731132935555"}]},"ts":"1731132935555"} 2024-11-09T06:15:35,560 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T06:15:35,562 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T06:15:35,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3060 sec 2024-11-09T06:15:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T06:15:36,434 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:15:36,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:15:36,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:15:36,442 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:15:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:15:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:15:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fa5eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:15:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:15:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:36,447 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:15:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a66f173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,449 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:36,455 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:36,459 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T06:15:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a84bcce7bad,37069,1731132928314 2024-11-09T06:15:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c52dcb8 2024-11-09T06:15:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T06:15:36,468 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T06:15:36,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:15:36,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:15:36,478 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:15:36,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:15:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:15:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d7efe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:15:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:15:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:36,490 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:15:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:15:36,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:15:36,498 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:15:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:15:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:15:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553a6f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:15:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:15:36,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:36,502 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:15:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc4f8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:15:36,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:15:36,509 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:15:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:15:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:15:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:15:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:15:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:36,514 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:15:36,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b0754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:15:36,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:15:36,522 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:15:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:15:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:15:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e855a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:15:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:15:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:36,527 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:15:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b24d866, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,530 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:15:36,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:15:36,532 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:15:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:15:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:15:36,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4331d0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:15:36,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:15:36,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:36,535 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:15:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17daddbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:15:36,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:15:36,542 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:15:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:15:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:15:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3ac0b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:15:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:15:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:36,547 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:15:36,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c90417c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:15:36,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:15:36,555 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:15:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:15:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:15:36,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f83cf40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:15:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:15:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:36,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:15:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605ecfde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:15:36,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:15:36,567 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:15:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:15:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:15:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b12d099, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:15:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:15:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:36,570 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492f9788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:15:36,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:15:36,575 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:15:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:15:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:15:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684b04c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:15:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:15:36,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:36,578 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:15:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d87de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:15:36,587 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:36,589 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:36,589 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:36,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:36,598 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:36,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:15:36,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:15:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:15:36,609 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:36,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:15:36,626 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:36,630 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:36,631 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:36,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:15:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:36,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:36,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:15:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:15:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-09T06:15:36,664 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:15:36,665 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:36,667 DEBUG [Thread-147 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:36,667 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:36,668 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:15:36,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:36,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:15:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:15:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T06:15:36,678 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:36,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:15:36,697 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:15:36,699 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:15:36,700 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:15:36,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:15:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:36,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:15:36,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:36,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:36,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:36,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:36,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:36,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T06:15:36,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:36,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:36,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:36,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:36,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/10a640b45e1843bdb759b84f2dff96e3 is 50, key is test_row_1/A:col10/1731132936732/Put/seqid=0 2024-11-09T06:15:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T06:15:37,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741837_1013 (size=16677) 2024-11-09T06:15:37,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731132996915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731132996975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731132996993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731132996996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731132997022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:37,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731132997194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731132997196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731132997199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731132997199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731132997200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:37,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:37,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T06:15:37,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/10a640b45e1843bdb759b84f2dff96e3 2024-11-09T06:15:37,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:37,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:37,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731132997418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731132997418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731132997418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731132997420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731132997418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/f251190d659344c9983584f6fe96fc8c is 50, key is test_row_1/B:col10/1731132936732/Put/seqid=0 2024-11-09T06:15:37,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741838_1014 (size=9657) 2024-11-09T06:15:37,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:37,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:37,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/f251190d659344c9983584f6fe96fc8c 2024-11-09T06:15:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/a846b8b826644140a1716b58fd6938f7 is 50, key is test_row_1/C:col10/1731132936732/Put/seqid=0 2024-11-09T06:15:37,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741839_1015 (size=9657) 2024-11-09T06:15:37,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/a846b8b826644140a1716b58fd6938f7 2024-11-09T06:15:37,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/10a640b45e1843bdb759b84f2dff96e3 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/10a640b45e1843bdb759b84f2dff96e3 2024-11-09T06:15:37,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/10a640b45e1843bdb759b84f2dff96e3, entries=250, sequenceid=13, filesize=16.3 K 2024-11-09T06:15:37,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/f251190d659344c9983584f6fe96fc8c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/f251190d659344c9983584f6fe96fc8c 2024-11-09T06:15:37,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:37,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:37,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/f251190d659344c9983584f6fe96fc8c, entries=100, sequenceid=13, filesize=9.4 K 2024-11-09T06:15:37,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731132997745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731132997746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731132997747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/a846b8b826644140a1716b58fd6938f7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/a846b8b826644140a1716b58fd6938f7 2024-11-09T06:15:37,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731132997747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:37,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/a846b8b826644140a1716b58fd6938f7, entries=100, sequenceid=13, filesize=9.4 K 2024-11-09T06:15:37,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4cec2ccd1a271d540c1f369d10f41e7f in 1044ms, sequenceid=13, compaction requested=false 2024-11-09T06:15:37,795 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T06:15:37,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T06:15:37,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132936907, max timestamp=1731132937792Num uniques 859; 2024-11-09T06:15:37,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:15:37,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132936907, max timestamp=1731132937792Num uniques 859; 2024-11-09T06:15:37,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:15:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:37,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-09T06:15:37,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:37,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:15:37,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:37,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:15:37,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:37,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:15:37,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/2307cc57e7f349efb3b0ee7eba0e4b23 is 50, key is test_row_0/A:col10/1731132936971/Put/seqid=0 2024-11-09T06:15:37,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741840_1016 (size=14341) 2024-11-09T06:15:37,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/2307cc57e7f349efb3b0ee7eba0e4b23 2024-11-09T06:15:37,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:37,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:37,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/7fa26b975ab6424ba4e9167d07a9bf7b is 50, key is test_row_0/B:col10/1731132936971/Put/seqid=0 2024-11-09T06:15:37,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741841_1017 (size=12001) 2024-11-09T06:15:38,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:38,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:38,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:38,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:38,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:38,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:38,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:38,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:38,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132938153, max timestamp=1731132938262Num uniques 859; 2024-11-09T06:15:38,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:15:38,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132938153, max timestamp=1731132938262Num uniques 859; 2024-11-09T06:15:38,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:15:38,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132938153, max timestamp=1731132938262Num uniques 859; 2024-11-09T06:15:38,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:15:38,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/7fa26b975ab6424ba4e9167d07a9bf7b 2024-11-09T06:15:38,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731132998355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731132998349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731132998357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731132998360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:38,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:38,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:38,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731132998371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,390 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T06:15:38,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/c94b7164c0074958a197b0314ea502fe is 50, key is test_row_0/C:col10/1731132936971/Put/seqid=0 2024-11-09T06:15:38,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741842_1018 (size=12001) 2024-11-09T06:15:38,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/c94b7164c0074958a197b0314ea502fe 2024-11-09T06:15:38,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731132998489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/2307cc57e7f349efb3b0ee7eba0e4b23 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2307cc57e7f349efb3b0ee7eba0e4b23 2024-11-09T06:15:38,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731132998493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731132998505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731132998498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731132998506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:38,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:38,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2307cc57e7f349efb3b0ee7eba0e4b23, entries=200, sequenceid=34, filesize=14.0 K 2024-11-09T06:15:38,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/7fa26b975ab6424ba4e9167d07a9bf7b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7fa26b975ab6424ba4e9167d07a9bf7b 2024-11-09T06:15:38,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7fa26b975ab6424ba4e9167d07a9bf7b, entries=150, sequenceid=34, filesize=11.7 K 2024-11-09T06:15:38,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/c94b7164c0074958a197b0314ea502fe as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c94b7164c0074958a197b0314ea502fe 2024-11-09T06:15:38,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c94b7164c0074958a197b0314ea502fe, entries=150, sequenceid=34, filesize=11.7 K 2024-11-09T06:15:38,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 4cec2ccd1a271d540c1f369d10f41e7f in 786ms, sequenceid=34, compaction requested=false 2024-11-09T06:15:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:38,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T06:15:38,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:38,698 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-09T06:15:38,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:38,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:15:38,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:15:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:15:38,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/bc9bb7953f30483c93b84ece66ff26c9 is 50, key is test_row_0/A:col10/1731132938350/Put/seqid=0 2024-11-09T06:15:38,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:38,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741843_1019 (size=12001) 2024-11-09T06:15:38,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731132998782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731132998787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731132998790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731132998789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731132998806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T06:15:38,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731132998913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731132998914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731132998925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731132998925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731132998925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:39,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731132999133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731132999133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:39,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731132999144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:39,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731132999144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731132999144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:39,195 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/bc9bb7953f30483c93b84ece66ff26c9 2024-11-09T06:15:39,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/66019a6376674dfba3f8c102ebf9e6e6 is 50, key is test_row_0/B:col10/1731132938350/Put/seqid=0 2024-11-09T06:15:39,270 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T06:15:39,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741844_1020 (size=12001) 2024-11-09T06:15:39,280 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/66019a6376674dfba3f8c102ebf9e6e6 2024-11-09T06:15:39,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/572fb078844747bd9d216658a4001182 is 50, key is test_row_0/C:col10/1731132938350/Put/seqid=0 2024-11-09T06:15:39,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741845_1021 (size=12001) 2024-11-09T06:15:39,376 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/572fb078844747bd9d216658a4001182 2024-11-09T06:15:39,396 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T06:15:39,397 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-09T06:15:39,404 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T06:15:39,404 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-09T06:15:39,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-09T06:15:39,405 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-09T06:15:39,407 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T06:15:39,407 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-09T06:15:39,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/bc9bb7953f30483c93b84ece66ff26c9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/bc9bb7953f30483c93b84ece66ff26c9 2024-11-09T06:15:39,428 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/bc9bb7953f30483c93b84ece66ff26c9, entries=150, sequenceid=70, filesize=11.7 K 2024-11-09T06:15:39,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/66019a6376674dfba3f8c102ebf9e6e6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/66019a6376674dfba3f8c102ebf9e6e6 2024-11-09T06:15:39,450 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/66019a6376674dfba3f8c102ebf9e6e6, entries=150, sequenceid=70, filesize=11.7 K 2024-11-09T06:15:39,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/572fb078844747bd9d216658a4001182 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/572fb078844747bd9d216658a4001182 2024-11-09T06:15:39,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731132999454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:39,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731132999455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:39,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731132999463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:39,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731132999464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:39,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731132999464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:39,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:39,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:39,489 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/572fb078844747bd9d216658a4001182, entries=150, sequenceid=70, filesize=11.7 K 2024-11-09T06:15:39,491 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 4cec2ccd1a271d540c1f369d10f41e7f in 793ms, sequenceid=70, compaction requested=true 2024-11-09T06:15:39,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:39,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:39,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-09T06:15:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-09T06:15:39,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-09T06:15:39,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8270 sec 2024-11-09T06:15:39,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.8580 sec 2024-11-09T06:15:39,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T06:15:39,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:39,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:39,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:39,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:39,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:39,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:39,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/a0cad2c2bd6a4285b3fc56e43701e094 is 50, key is test_row_0/A:col10/1731132938785/Put/seqid=0 2024-11-09T06:15:40,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741846_1022 (size=14341) 2024-11-09T06:15:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133000042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133000044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133000046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133000047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133000049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133000164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133000164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133000164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133000169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133000174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133000384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133000385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133000387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133000387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133000393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/a0cad2c2bd6a4285b3fc56e43701e094 2024-11-09T06:15:40,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/5bf84afedabe4e2fa634fd642cfa16ce is 50, key is test_row_0/B:col10/1731132938785/Put/seqid=0 2024-11-09T06:15:40,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741847_1023 (size=12001) 2024-11-09T06:15:40,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/5bf84afedabe4e2fa634fd642cfa16ce 2024-11-09T06:15:40,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/ff069f01e205472e9ca8573817b80b8f is 50, key is test_row_0/C:col10/1731132938785/Put/seqid=0 2024-11-09T06:15:40,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741848_1024 (size=12001) 2024-11-09T06:15:40,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/ff069f01e205472e9ca8573817b80b8f 2024-11-09T06:15:40,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/a0cad2c2bd6a4285b3fc56e43701e094 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/a0cad2c2bd6a4285b3fc56e43701e094 2024-11-09T06:15:40,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/a0cad2c2bd6a4285b3fc56e43701e094, entries=200, sequenceid=82, filesize=14.0 K 2024-11-09T06:15:40,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/5bf84afedabe4e2fa634fd642cfa16ce as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/5bf84afedabe4e2fa634fd642cfa16ce 2024-11-09T06:15:40,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/5bf84afedabe4e2fa634fd642cfa16ce, entries=150, sequenceid=82, filesize=11.7 K 2024-11-09T06:15:40,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/ff069f01e205472e9ca8573817b80b8f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/ff069f01e205472e9ca8573817b80b8f 2024-11-09T06:15:40,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133000703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133000704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133000704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133000704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:40,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/ff069f01e205472e9ca8573817b80b8f, entries=150, sequenceid=82, filesize=11.7 K 2024-11-09T06:15:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133000713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:40,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4cec2ccd1a271d540c1f369d10f41e7f in 755ms, sequenceid=82, compaction requested=true 2024-11-09T06:15:40,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:40,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:15:40,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:40,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:15:40,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:40,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:15:40,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:40,748 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:15:40,749 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:15:40,755 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:15:40,757 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/B is initiating minor compaction (all files) 2024-11-09T06:15:40,758 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/B in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:40,758 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/f251190d659344c9983584f6fe96fc8c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7fa26b975ab6424ba4e9167d07a9bf7b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/66019a6376674dfba3f8c102ebf9e6e6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/5bf84afedabe4e2fa634fd642cfa16ce] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=44.6 K 2024-11-09T06:15:40,758 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:15:40,758 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/A is initiating minor compaction (all files) 2024-11-09T06:15:40,758 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/A in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:40,759 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/10a640b45e1843bdb759b84f2dff96e3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2307cc57e7f349efb3b0ee7eba0e4b23, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/bc9bb7953f30483c93b84ece66ff26c9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/a0cad2c2bd6a4285b3fc56e43701e094] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=56.0 K 2024-11-09T06:15:40,760 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10a640b45e1843bdb759b84f2dff96e3, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731132936695 2024-11-09T06:15:40,770 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2307cc57e7f349efb3b0ee7eba0e4b23, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731132936907 2024-11-09T06:15:40,772 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting f251190d659344c9983584f6fe96fc8c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731132936732 2024-11-09T06:15:40,773 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc9bb7953f30483c93b84ece66ff26c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731132938350 2024-11-09T06:15:40,774 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa26b975ab6424ba4e9167d07a9bf7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731132936907 2024-11-09T06:15:40,775 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 66019a6376674dfba3f8c102ebf9e6e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731132938350 2024-11-09T06:15:40,775 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0cad2c2bd6a4285b3fc56e43701e094, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731132938759 2024-11-09T06:15:40,777 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bf84afedabe4e2fa634fd642cfa16ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731132938759 2024-11-09T06:15:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T06:15:40,833 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:15:40,834 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:15:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-09T06:15:40,839 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:15:40,845 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#A#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:40,846 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#B#compaction#12 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T06:15:40,847 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/9e6d90e9db654d8e82882438966a920e is 50, key is test_row_0/B:col10/1731132938785/Put/seqid=0 2024-11-09T06:15:40,847 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/7bead93fd3d84f55b6add64a389eff0f is 50, key is test_row_0/A:col10/1731132938785/Put/seqid=0 2024-11-09T06:15:40,847 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:15:40,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:15:40,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741850_1026 (size=12139) 2024-11-09T06:15:40,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741849_1025 (size=12139) 2024-11-09T06:15:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T06:15:40,969 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/7bead93fd3d84f55b6add64a389eff0f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/7bead93fd3d84f55b6add64a389eff0f 2024-11-09T06:15:41,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T06:15:41,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:41,004 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:15:41,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:41,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:41,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:41,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:41,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:41,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:41,018 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/A of 4cec2ccd1a271d540c1f369d10f41e7f into 7bead93fd3d84f55b6add64a389eff0f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:41,018 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:41,022 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/A, priority=12, startTime=1731132940743; duration=0sec 2024-11-09T06:15:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/48445f1ad89c4a319e348996ba7ee2f6 is 50, key is test_row_0/A:col10/1731132940042/Put/seqid=0 2024-11-09T06:15:41,032 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:41,032 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:A 2024-11-09T06:15:41,032 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:15:41,036 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:15:41,037 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/C is initiating minor compaction (all files) 2024-11-09T06:15:41,037 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/C in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:41,037 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/a846b8b826644140a1716b58fd6938f7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c94b7164c0074958a197b0314ea502fe, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/572fb078844747bd9d216658a4001182, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/ff069f01e205472e9ca8573817b80b8f] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=44.6 K 2024-11-09T06:15:41,042 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a846b8b826644140a1716b58fd6938f7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731132936732 2024-11-09T06:15:41,044 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c94b7164c0074958a197b0314ea502fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731132936907 2024-11-09T06:15:41,045 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 572fb078844747bd9d216658a4001182, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731132938350 2024-11-09T06:15:41,046 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff069f01e205472e9ca8573817b80b8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731132938759 2024-11-09T06:15:41,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741851_1027 (size=12001) 2024-11-09T06:15:41,074 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/48445f1ad89c4a319e348996ba7ee2f6 2024-11-09T06:15:41,084 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#C#compaction#15 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:41,085 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/557a13bedfdc4beb9339b7df26db8c92 is 50, key is test_row_0/C:col10/1731132938785/Put/seqid=0 2024-11-09T06:15:41,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741852_1028 (size=12139) 2024-11-09T06:15:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/1d94775cc07245478ea60e85a05108ea is 50, key is test_row_0/B:col10/1731132940042/Put/seqid=0 2024-11-09T06:15:41,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741853_1029 (size=12001) 2024-11-09T06:15:41,148 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/1d94775cc07245478ea60e85a05108ea 2024-11-09T06:15:41,159 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/557a13bedfdc4beb9339b7df26db8c92 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/557a13bedfdc4beb9339b7df26db8c92 2024-11-09T06:15:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T06:15:41,189 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/C of 4cec2ccd1a271d540c1f369d10f41e7f into 557a13bedfdc4beb9339b7df26db8c92(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:41,190 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:41,190 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/C, priority=12, startTime=1731132940746; duration=0sec 2024-11-09T06:15:41,190 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:41,190 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:C 2024-11-09T06:15:41,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/f8a06a719f274889ac66f3150c2eab5c is 50, key is test_row_0/C:col10/1731132940042/Put/seqid=0 2024-11-09T06:15:41,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:41,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741854_1030 (size=12001) 2024-11-09T06:15:41,261 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/f8a06a719f274889ac66f3150c2eab5c 2024-11-09T06:15:41,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133001257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133001264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/48445f1ad89c4a319e348996ba7ee2f6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/48445f1ad89c4a319e348996ba7ee2f6 2024-11-09T06:15:41,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133001276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133001278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133001276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,306 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/48445f1ad89c4a319e348996ba7ee2f6, entries=150, sequenceid=103, filesize=11.7 K 2024-11-09T06:15:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/1d94775cc07245478ea60e85a05108ea as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/1d94775cc07245478ea60e85a05108ea 2024-11-09T06:15:41,324 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/1d94775cc07245478ea60e85a05108ea, entries=150, sequenceid=103, filesize=11.7 K 2024-11-09T06:15:41,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/f8a06a719f274889ac66f3150c2eab5c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f8a06a719f274889ac66f3150c2eab5c 2024-11-09T06:15:41,344 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f8a06a719f274889ac66f3150c2eab5c, entries=150, sequenceid=103, filesize=11.7 K 2024-11-09T06:15:41,350 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 4cec2ccd1a271d540c1f369d10f41e7f in 346ms, sequenceid=103, compaction requested=false 2024-11-09T06:15:41,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:41,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:41,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-09T06:15:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-09T06:15:41,368 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/9e6d90e9db654d8e82882438966a920e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/9e6d90e9db654d8e82882438966a920e 2024-11-09T06:15:41,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-09T06:15:41,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 515 msec 2024-11-09T06:15:41,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 549 msec 2024-11-09T06:15:41,393 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/B of 4cec2ccd1a271d540c1f369d10f41e7f into 9e6d90e9db654d8e82882438966a920e(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:41,393 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:41,393 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/B, priority=12, startTime=1731132940746; duration=0sec 2024-11-09T06:15:41,394 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:41,394 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:B 2024-11-09T06:15:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:41,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T06:15:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:41,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:41,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/d386b838052949368607e251de4b6413 is 50, key is test_row_0/A:col10/1731132941272/Put/seqid=0 2024-11-09T06:15:41,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741855_1031 (size=14341) 2024-11-09T06:15:41,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133001437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133001439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133001441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133001445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133001451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T06:15:41,489 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:15:41,490 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:15:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-09T06:15:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T06:15:41,502 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:15:41,510 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:15:41,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:15:41,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133001563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133001564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133001564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133001564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133001564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T06:15:41,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:41,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:41,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:41,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:41,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:41,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:41,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133001783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133001784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133001784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133001784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133001786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:41,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/d386b838052949368607e251de4b6413 2024-11-09T06:15:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T06:15:41,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:41,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:41,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:41,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:41,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:41,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:41,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/956f763c38cf4d898304cb50a21662ea is 50, key is test_row_0/B:col10/1731132941272/Put/seqid=0 2024-11-09T06:15:41,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741856_1032 (size=12001) 2024-11-09T06:15:41,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:41,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:41,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:41,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:41,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:41,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133002093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133002093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133002093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133002103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133002103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T06:15:42,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:42,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:42,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/956f763c38cf4d898304cb50a21662ea 2024-11-09T06:15:42,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/2d4239b7e6494f30891ae6676dcebf75 is 50, key is test_row_0/C:col10/1731132941272/Put/seqid=0 2024-11-09T06:15:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741857_1033 (size=12001) 2024-11-09T06:15:42,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/2d4239b7e6494f30891ae6676dcebf75 2024-11-09T06:15:42,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/d386b838052949368607e251de4b6413 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d386b838052949368607e251de4b6413 2024-11-09T06:15:42,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d386b838052949368607e251de4b6413, entries=200, sequenceid=119, filesize=14.0 K 2024-11-09T06:15:42,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/956f763c38cf4d898304cb50a21662ea as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/956f763c38cf4d898304cb50a21662ea 2024-11-09T06:15:42,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/956f763c38cf4d898304cb50a21662ea, entries=150, sequenceid=119, filesize=11.7 K 2024-11-09T06:15:42,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/2d4239b7e6494f30891ae6676dcebf75 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2d4239b7e6494f30891ae6676dcebf75 2024-11-09T06:15:42,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:42,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:42,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2d4239b7e6494f30891ae6676dcebf75, entries=150, sequenceid=119, filesize=11.7 K 2024-11-09T06:15:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4cec2ccd1a271d540c1f369d10f41e7f in 1081ms, sequenceid=119, compaction requested=true 2024-11-09T06:15:42,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:42,476 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:42,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:15:42,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:42,476 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:42,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:15:42,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:42,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:15:42,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:42,487 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:42,487 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/B is initiating minor compaction (all files) 2024-11-09T06:15:42,487 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/B in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,488 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/9e6d90e9db654d8e82882438966a920e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/1d94775cc07245478ea60e85a05108ea, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/956f763c38cf4d898304cb50a21662ea] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=35.3 K 2024-11-09T06:15:42,488 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:42,488 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/A is initiating minor compaction (all files) 2024-11-09T06:15:42,488 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/A in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,488 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/7bead93fd3d84f55b6add64a389eff0f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/48445f1ad89c4a319e348996ba7ee2f6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d386b838052949368607e251de4b6413] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=37.6 K 2024-11-09T06:15:42,490 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e6d90e9db654d8e82882438966a920e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731132938759 2024-11-09T06:15:42,491 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d94775cc07245478ea60e85a05108ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731132940029 2024-11-09T06:15:42,491 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bead93fd3d84f55b6add64a389eff0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731132938759 2024-11-09T06:15:42,492 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 956f763c38cf4d898304cb50a21662ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731132941253 2024-11-09T06:15:42,493 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48445f1ad89c4a319e348996ba7ee2f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731132940029 2024-11-09T06:15:42,494 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d386b838052949368607e251de4b6413, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731132941253 2024-11-09T06:15:42,535 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:42,536 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/61182a4c7750450b9f7a9243a9ac93bd is 50, key is test_row_0/B:col10/1731132941272/Put/seqid=0 2024-11-09T06:15:42,548 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:42,549 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/d0ba71543ee94bf09e2806a13cf65dee is 50, key is test_row_0/A:col10/1731132941272/Put/seqid=0 2024-11-09T06:15:42,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741858_1034 (size=12241) 2024-11-09T06:15:42,612 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/61182a4c7750450b9f7a9243a9ac93bd as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/61182a4c7750450b9f7a9243a9ac93bd 2024-11-09T06:15:42,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741859_1035 (size=12241) 2024-11-09T06:15:42,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T06:15:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:42,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:42,628 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/d0ba71543ee94bf09e2806a13cf65dee as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d0ba71543ee94bf09e2806a13cf65dee 2024-11-09T06:15:42,628 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/B of 4cec2ccd1a271d540c1f369d10f41e7f into 61182a4c7750450b9f7a9243a9ac93bd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:42,628 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:42,628 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/B, priority=13, startTime=1731132942476; duration=0sec 2024-11-09T06:15:42,629 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:42,629 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:B 2024-11-09T06:15:42,629 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:42,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:42,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:42,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,636 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:42,636 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/C is initiating minor compaction (all files) 2024-11-09T06:15:42,636 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/C in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,637 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/557a13bedfdc4beb9339b7df26db8c92, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f8a06a719f274889ac66f3150c2eab5c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2d4239b7e6494f30891ae6676dcebf75] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=35.3 K 2024-11-09T06:15:42,639 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 557a13bedfdc4beb9339b7df26db8c92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731132938759 2024-11-09T06:15:42,640 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting f8a06a719f274889ac66f3150c2eab5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731132940029 2024-11-09T06:15:42,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/9e1574b7b69542e995f70817894194bc is 50, key is test_row_0/A:col10/1731132942616/Put/seqid=0 2024-11-09T06:15:42,642 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d4239b7e6494f30891ae6676dcebf75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731132941253 2024-11-09T06:15:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T06:15:42,648 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/A of 4cec2ccd1a271d540c1f369d10f41e7f into d0ba71543ee94bf09e2806a13cf65dee(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:42,648 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:42,648 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/A, priority=13, startTime=1731132942476; duration=0sec 2024-11-09T06:15:42,649 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:42,649 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:A 2024-11-09T06:15:42,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133002652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133002662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133002666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133002666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,672 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#C#compaction#24 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:42,673 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e2169ea7bf21469488abf508c6529b6b is 50, key is test_row_0/C:col10/1731132941272/Put/seqid=0 2024-11-09T06:15:42,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133002668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741860_1036 (size=16931) 2024-11-09T06:15:42,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/9e1574b7b69542e995f70817894194bc 2024-11-09T06:15:42,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/9480dfe0e6f54868a662d9b4e577a764 is 50, key is test_row_0/B:col10/1731132942616/Put/seqid=0 2024-11-09T06:15:42,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741861_1037 (size=12241) 2024-11-09T06:15:42,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741862_1038 (size=12151) 2024-11-09T06:15:42,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133002773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133002773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133002783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133002784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:42,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:42,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133002795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:42,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:42,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:42,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133002983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:42,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133002993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133002984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133003000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133003013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:43,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:43,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,151 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e2169ea7bf21469488abf508c6529b6b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e2169ea7bf21469488abf508c6529b6b 2024-11-09T06:15:43,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/9480dfe0e6f54868a662d9b4e577a764 2024-11-09T06:15:43,167 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/C of 4cec2ccd1a271d540c1f369d10f41e7f into e2169ea7bf21469488abf508c6529b6b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:43,168 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:43,168 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/C, priority=13, startTime=1731132942477; duration=0sec 2024-11-09T06:15:43,168 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:43,168 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:C 2024-11-09T06:15:43,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/eb6cea8d2ac04539b3d43f59d7131d06 is 50, key is test_row_0/C:col10/1731132942616/Put/seqid=0 2024-11-09T06:15:43,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741863_1039 (size=12151) 2024-11-09T06:15:43,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/eb6cea8d2ac04539b3d43f59d7131d06 2024-11-09T06:15:43,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/9e1574b7b69542e995f70817894194bc as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/9e1574b7b69542e995f70817894194bc 2024-11-09T06:15:43,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/9e1574b7b69542e995f70817894194bc, entries=250, sequenceid=139, filesize=16.5 K 2024-11-09T06:15:43,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:43,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:43,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/9480dfe0e6f54868a662d9b4e577a764 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/9480dfe0e6f54868a662d9b4e577a764 2024-11-09T06:15:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/9480dfe0e6f54868a662d9b4e577a764, entries=150, sequenceid=139, filesize=11.9 K 2024-11-09T06:15:43,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/eb6cea8d2ac04539b3d43f59d7131d06 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/eb6cea8d2ac04539b3d43f59d7131d06 2024-11-09T06:15:43,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133003293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/eb6cea8d2ac04539b3d43f59d7131d06, entries=150, sequenceid=139, filesize=11.9 K 2024-11-09T06:15:43,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133003317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 4cec2ccd1a271d540c1f369d10f41e7f in 708ms, sequenceid=139, compaction requested=false 2024-11-09T06:15:43,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:43,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T06:15:43,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:43,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:43,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:43,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:43,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:43,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:43,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/b5a2ac599be943429f1c7b860e45caf7 is 50, key is test_row_0/A:col10/1731132942661/Put/seqid=0 2024-11-09T06:15:43,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741864_1040 (size=12151) 2024-11-09T06:15:43,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/b5a2ac599be943429f1c7b860e45caf7 2024-11-09T06:15:43,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133003465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133003465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133003467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/32744b241509465ead823e7510b7add3 is 50, key is test_row_0/B:col10/1731132942661/Put/seqid=0 2024-11-09T06:15:43,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741865_1041 (size=12151) 2024-11-09T06:15:43,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:43,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:43,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133003583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133003584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133003584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T06:15:43,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133003794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133003795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133003802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133003834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133003834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:43,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:43,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:43,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:43,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/32744b241509465ead823e7510b7add3 2024-11-09T06:15:43,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e24d31a2abfa45a591359951346601c4 is 50, key is test_row_0/C:col10/1731132942661/Put/seqid=0 2024-11-09T06:15:44,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741866_1042 (size=12151) 2024-11-09T06:15:44,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e24d31a2abfa45a591359951346601c4 2024-11-09T06:15:44,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/b5a2ac599be943429f1c7b860e45caf7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b5a2ac599be943429f1c7b860e45caf7 2024-11-09T06:15:44,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:44,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b5a2ac599be943429f1c7b860e45caf7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T06:15:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/32744b241509465ead823e7510b7add3 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/32744b241509465ead823e7510b7add3 2024-11-09T06:15:44,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/32744b241509465ead823e7510b7add3, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T06:15:44,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e24d31a2abfa45a591359951346601c4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e24d31a2abfa45a591359951346601c4 2024-11-09T06:15:44,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e24d31a2abfa45a591359951346601c4, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T06:15:44,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4cec2ccd1a271d540c1f369d10f41e7f in 797ms, sequenceid=156, compaction requested=true 2024-11-09T06:15:44,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:44,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:15:44,127 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:44,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:44,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:15:44,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:44,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:15:44,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T06:15:44,128 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:44,131 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41323 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:44,131 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/A is initiating minor compaction (all files) 2024-11-09T06:15:44,131 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/A in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,131 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:44,132 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/B is initiating minor compaction (all files) 2024-11-09T06:15:44,132 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d0ba71543ee94bf09e2806a13cf65dee, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/9e1574b7b69542e995f70817894194bc, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b5a2ac599be943429f1c7b860e45caf7] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=40.4 K 2024-11-09T06:15:44,132 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/B in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,132 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/61182a4c7750450b9f7a9243a9ac93bd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/9480dfe0e6f54868a662d9b4e577a764, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/32744b241509465ead823e7510b7add3] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=35.7 K 2024-11-09T06:15:44,133 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting d0ba71543ee94bf09e2806a13cf65dee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731132941253 2024-11-09T06:15:44,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T06:15:44,133 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61182a4c7750450b9f7a9243a9ac93bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731132941253 2024-11-09T06:15:44,134 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e1574b7b69542e995f70817894194bc, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731132941434 2024-11-09T06:15:44,134 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9480dfe0e6f54868a662d9b4e577a764, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731132941446 2024-11-09T06:15:44,134 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a2ac599be943429f1c7b860e45caf7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731132942661 2024-11-09T06:15:44,135 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32744b241509465ead823e7510b7add3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731132942661 2024-11-09T06:15:44,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:44,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:44,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:44,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:44,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:44,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:44,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/c6f157f3c80a4ad8aee78cf70ba7fa4e is 50, key is test_row_0/A:col10/1731132943438/Put/seqid=0 2024-11-09T06:15:44,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:44,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:44,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741867_1043 (size=16931) 2024-11-09T06:15:44,251 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:44,252 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/d3df944ed84a463ca3630620f334fc72 is 50, key is test_row_0/B:col10/1731132942661/Put/seqid=0 2024-11-09T06:15:44,274 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#A#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:44,275 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/076d1959c98944d2bd5758080f42c90b is 50, key is test_row_0/A:col10/1731132942661/Put/seqid=0 2024-11-09T06:15:44,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741868_1044 (size=12493) 2024-11-09T06:15:44,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133004321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:44,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741869_1045 (size=12493) 2024-11-09T06:15:44,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133004330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133004337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:44,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:44,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:44,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133004453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:44,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133004463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:44,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:44,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133004477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:44,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:44,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:44,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/c6f157f3c80a4ad8aee78cf70ba7fa4e 2024-11-09T06:15:44,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/8046f82778c140ccb255ee42082d0945 is 50, key is test_row_0/B:col10/1731132943438/Put/seqid=0 2024-11-09T06:15:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133004675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:44,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133004692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:44,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133004700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:44,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741870_1046 (size=12151) 2024-11-09T06:15:44,754 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/d3df944ed84a463ca3630620f334fc72 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d3df944ed84a463ca3630620f334fc72 2024-11-09T06:15:44,778 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/B of 4cec2ccd1a271d540c1f369d10f41e7f into d3df944ed84a463ca3630620f334fc72(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:44,778 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:44,778 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/B, priority=13, startTime=1731132944127; duration=0sec 2024-11-09T06:15:44,779 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:44,779 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:B 2024-11-09T06:15:44,779 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:44,779 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/076d1959c98944d2bd5758080f42c90b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/076d1959c98944d2bd5758080f42c90b 2024-11-09T06:15:44,781 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:44,781 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/C is initiating minor compaction (all files) 2024-11-09T06:15:44,782 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/C in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,782 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e2169ea7bf21469488abf508c6529b6b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/eb6cea8d2ac04539b3d43f59d7131d06, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e24d31a2abfa45a591359951346601c4] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=35.7 K 2024-11-09T06:15:44,783 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2169ea7bf21469488abf508c6529b6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731132941253 2024-11-09T06:15:44,784 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb6cea8d2ac04539b3d43f59d7131d06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731132941446 2024-11-09T06:15:44,785 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e24d31a2abfa45a591359951346601c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731132942661 2024-11-09T06:15:44,797 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/A of 4cec2ccd1a271d540c1f369d10f41e7f into 076d1959c98944d2bd5758080f42c90b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:44,798 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:44,799 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/A, priority=13, startTime=1731132944127; duration=0sec 2024-11-09T06:15:44,799 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:44,799 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:A 2024-11-09T06:15:44,825 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#C#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:44,826 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/7682e22a57274f8694fe22109a6a3cb9 is 50, key is test_row_0/C:col10/1731132942661/Put/seqid=0 2024-11-09T06:15:44,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:44,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:44,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:44,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:44,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133004854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:44,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133004853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:44,877 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in 4cec2ccd1a271d540c1f369d10f41e7f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:15:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:44,874 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in 4cec2ccd1a271d540c1f369d10f41e7f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:15:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:44,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741871_1047 (size=12493) 2024-11-09T06:15:44,904 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/7682e22a57274f8694fe22109a6a3cb9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/7682e22a57274f8694fe22109a6a3cb9 2024-11-09T06:15:44,918 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/C of 4cec2ccd1a271d540c1f369d10f41e7f into 7682e22a57274f8694fe22109a6a3cb9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:44,918 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:44,918 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/C, priority=13, startTime=1731132944128; duration=0sec 2024-11-09T06:15:44,919 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:44,919 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:C 2024-11-09T06:15:45,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:45,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:45,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:45,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:45,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:45,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:45,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133005004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133005004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:45,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133005023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:45,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/8046f82778c140ccb255ee42082d0945 2024-11-09T06:15:45,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/73e4a19b9af14140adcbf4a699504f4c is 50, key is test_row_0/C:col10/1731132943438/Put/seqid=0 2024-11-09T06:15:45,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:45,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:45,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:45,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:45,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:45,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:45,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741872_1048 (size=12151) 2024-11-09T06:15:45,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/73e4a19b9af14140adcbf4a699504f4c 2024-11-09T06:15:45,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/c6f157f3c80a4ad8aee78cf70ba7fa4e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/c6f157f3c80a4ad8aee78cf70ba7fa4e 2024-11-09T06:15:45,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/c6f157f3c80a4ad8aee78cf70ba7fa4e, entries=250, sequenceid=176, filesize=16.5 K 2024-11-09T06:15:45,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/8046f82778c140ccb255ee42082d0945 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/8046f82778c140ccb255ee42082d0945 2024-11-09T06:15:45,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/8046f82778c140ccb255ee42082d0945, entries=150, sequenceid=176, filesize=11.9 K 2024-11-09T06:15:45,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/73e4a19b9af14140adcbf4a699504f4c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/73e4a19b9af14140adcbf4a699504f4c 2024-11-09T06:15:45,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:45,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:45,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:45,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:45,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:45,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:45,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/73e4a19b9af14140adcbf4a699504f4c, entries=150, sequenceid=176, filesize=11.9 K 2024-11-09T06:15:45,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 4cec2ccd1a271d540c1f369d10f41e7f in 1235ms, sequenceid=176, compaction requested=false 2024-11-09T06:15:45,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:45,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T06:15:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:45,504 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:15:45,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:45,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:45,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/70de94b114794474a34dbff5e2a5827a is 50, key is test_row_0/A:col10/1731132944173/Put/seqid=0 2024-11-09T06:15:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:45,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:45,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741873_1049 (size=12151) 2024-11-09T06:15:45,552 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/70de94b114794474a34dbff5e2a5827a 2024-11-09T06:15:45,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4a8e06931eeb48d19719d2cc7a7cab0d is 50, key is test_row_0/B:col10/1731132944173/Put/seqid=0 2024-11-09T06:15:45,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741874_1050 (size=12151) 2024-11-09T06:15:45,621 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4a8e06931eeb48d19719d2cc7a7cab0d 2024-11-09T06:15:45,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e5a69727a94b49ebb3deac2b7fa6ad09 is 50, key is test_row_0/C:col10/1731132944173/Put/seqid=0 2024-11-09T06:15:45,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133005645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133005648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:45,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:45,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133005651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T06:15:45,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741875_1051 (size=12151) 2024-11-09T06:15:45,697 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e5a69727a94b49ebb3deac2b7fa6ad09 2024-11-09T06:15:45,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/70de94b114794474a34dbff5e2a5827a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/70de94b114794474a34dbff5e2a5827a 2024-11-09T06:15:45,725 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/70de94b114794474a34dbff5e2a5827a, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T06:15:45,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4a8e06931eeb48d19719d2cc7a7cab0d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4a8e06931eeb48d19719d2cc7a7cab0d 2024-11-09T06:15:45,741 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4a8e06931eeb48d19719d2cc7a7cab0d, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T06:15:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e5a69727a94b49ebb3deac2b7fa6ad09 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a69727a94b49ebb3deac2b7fa6ad09 2024-11-09T06:15:45,762 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a69727a94b49ebb3deac2b7fa6ad09, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T06:15:45,766 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4cec2ccd1a271d540c1f369d10f41e7f in 261ms, sequenceid=192, compaction requested=true 2024-11-09T06:15:45,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:45,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:45,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-09T06:15:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-09T06:15:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:45,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:15:45,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:45,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:45,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:45,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:45,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:45,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:45,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-09T06:15:45,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.2580 sec 2024-11-09T06:15:45,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 4.2890 sec 2024-11-09T06:15:45,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/1c5825c9b6fd455191e7a17941051a74 is 50, key is test_row_0/A:col10/1731132945647/Put/seqid=0 2024-11-09T06:15:45,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133005815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:45,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741876_1052 (size=14541) 2024-11-09T06:15:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133005820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:45,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133005833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:45,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133005943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:45,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133005953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:45,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133005963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:46,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133006152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:46,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133006163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133006173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:46,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/1c5825c9b6fd455191e7a17941051a74 2024-11-09T06:15:46,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4d881a3c916241878c6ac395b7bcd37a is 50, key is test_row_0/B:col10/1731132945647/Put/seqid=0 2024-11-09T06:15:46,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741877_1053 (size=12151) 2024-11-09T06:15:46,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4d881a3c916241878c6ac395b7bcd37a 2024-11-09T06:15:46,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/c7dbc3d2fb4345f080b690e6dba261c6 is 50, key is test_row_0/C:col10/1731132945647/Put/seqid=0 2024-11-09T06:15:46,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741878_1054 (size=12151) 2024-11-09T06:15:46,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/c7dbc3d2fb4345f080b690e6dba261c6 2024-11-09T06:15:46,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/1c5825c9b6fd455191e7a17941051a74 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1c5825c9b6fd455191e7a17941051a74 2024-11-09T06:15:46,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1c5825c9b6fd455191e7a17941051a74, entries=200, sequenceid=212, filesize=14.2 K 2024-11-09T06:15:46,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4d881a3c916241878c6ac395b7bcd37a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4d881a3c916241878c6ac395b7bcd37a 2024-11-09T06:15:46,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4d881a3c916241878c6ac395b7bcd37a, entries=150, sequenceid=212, filesize=11.9 K 2024-11-09T06:15:46,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133006463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:46,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/c7dbc3d2fb4345f080b690e6dba261c6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c7dbc3d2fb4345f080b690e6dba261c6 2024-11-09T06:15:46,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:46,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133006473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:46,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c7dbc3d2fb4345f080b690e6dba261c6, entries=150, sequenceid=212, filesize=11.9 K 2024-11-09T06:15:46,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133006484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:46,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4cec2ccd1a271d540c1f369d10f41e7f in 719ms, sequenceid=212, compaction requested=true 2024-11-09T06:15:46,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:46,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:15:46,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:46,491 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:15:46,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:15:46,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:46,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:15:46,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T06:15:46,492 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:15:46,500 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:15:46,500 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/A is initiating minor compaction (all files) 2024-11-09T06:15:46,500 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/A in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:46,500 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/076d1959c98944d2bd5758080f42c90b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/c6f157f3c80a4ad8aee78cf70ba7fa4e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/70de94b114794474a34dbff5e2a5827a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1c5825c9b6fd455191e7a17941051a74] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=54.8 K 2024-11-09T06:15:46,501 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:15:46,501 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/B is initiating minor compaction (all files) 2024-11-09T06:15:46,501 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/B in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:46,501 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d3df944ed84a463ca3630620f334fc72, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/8046f82778c140ccb255ee42082d0945, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4a8e06931eeb48d19719d2cc7a7cab0d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4d881a3c916241878c6ac395b7bcd37a] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=47.8 K 2024-11-09T06:15:46,502 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3df944ed84a463ca3630620f334fc72, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731132942661 2024-11-09T06:15:46,502 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 076d1959c98944d2bd5758080f42c90b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731132942661 2024-11-09T06:15:46,502 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8046f82778c140ccb255ee42082d0945, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731132943438 2024-11-09T06:15:46,502 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting c6f157f3c80a4ad8aee78cf70ba7fa4e, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731132943438 2024-11-09T06:15:46,503 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a8e06931eeb48d19719d2cc7a7cab0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731132944173 2024-11-09T06:15:46,503 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 70de94b114794474a34dbff5e2a5827a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731132944173 2024-11-09T06:15:46,503 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d881a3c916241878c6ac395b7bcd37a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731132945588 2024-11-09T06:15:46,504 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c5825c9b6fd455191e7a17941051a74, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731132945588 2024-11-09T06:15:46,533 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:46,533 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/a94e036140ae42ec8f0261257814e1d8 is 50, key is test_row_0/B:col10/1731132945647/Put/seqid=0 2024-11-09T06:15:46,543 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#A#compaction#43 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:46,544 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/89e85989c1724b66ad2bcbc6ed0e4ab6 is 50, key is test_row_0/A:col10/1731132945647/Put/seqid=0 2024-11-09T06:15:46,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741879_1055 (size=12629) 2024-11-09T06:15:46,599 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/a94e036140ae42ec8f0261257814e1d8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/a94e036140ae42ec8f0261257814e1d8 2024-11-09T06:15:46,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741880_1056 (size=12629) 2024-11-09T06:15:46,611 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/B of 4cec2ccd1a271d540c1f369d10f41e7f into a94e036140ae42ec8f0261257814e1d8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:46,611 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:46,611 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/B, priority=12, startTime=1731132946491; duration=0sec 2024-11-09T06:15:46,611 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:46,611 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:B 2024-11-09T06:15:46,612 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:15:46,613 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/89e85989c1724b66ad2bcbc6ed0e4ab6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/89e85989c1724b66ad2bcbc6ed0e4ab6 2024-11-09T06:15:46,614 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:15:46,615 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/C is initiating minor compaction (all files) 2024-11-09T06:15:46,615 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/C in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:46,615 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/7682e22a57274f8694fe22109a6a3cb9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/73e4a19b9af14140adcbf4a699504f4c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a69727a94b49ebb3deac2b7fa6ad09, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c7dbc3d2fb4345f080b690e6dba261c6] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=47.8 K 2024-11-09T06:15:46,616 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7682e22a57274f8694fe22109a6a3cb9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731132942661 2024-11-09T06:15:46,617 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73e4a19b9af14140adcbf4a699504f4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731132943438 2024-11-09T06:15:46,618 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5a69727a94b49ebb3deac2b7fa6ad09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731132944173 2024-11-09T06:15:46,618 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7dbc3d2fb4345f080b690e6dba261c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731132945588 2024-11-09T06:15:46,622 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/A of 4cec2ccd1a271d540c1f369d10f41e7f into 89e85989c1724b66ad2bcbc6ed0e4ab6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:46,623 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:46,623 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/A, priority=12, startTime=1731132946491; duration=0sec 2024-11-09T06:15:46,623 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:46,623 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:A 2024-11-09T06:15:46,643 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:46,644 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/107b0887f79d42528f0771059783cf3b is 50, key is test_row_0/C:col10/1731132945647/Put/seqid=0 2024-11-09T06:15:46,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741881_1057 (size=12629) 2024-11-09T06:15:46,689 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/107b0887f79d42528f0771059783cf3b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/107b0887f79d42528f0771059783cf3b 2024-11-09T06:15:46,700 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/C of 4cec2ccd1a271d540c1f369d10f41e7f into 107b0887f79d42528f0771059783cf3b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:46,700 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:46,700 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/C, priority=12, startTime=1731132946491; duration=0sec 2024-11-09T06:15:46,700 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:46,700 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:C 2024-11-09T06:15:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:46,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T06:15:46,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:46,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/0b8a9b102c9b4a0eb3d10d2fc7256a36 is 50, key is test_row_0/A:col10/1731132945828/Put/seqid=0 2024-11-09T06:15:46,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741882_1058 (size=12151) 2024-11-09T06:15:46,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/0b8a9b102c9b4a0eb3d10d2fc7256a36 2024-11-09T06:15:46,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/772ac1c17672464d82542a9afb84c37c is 50, key is test_row_0/B:col10/1731132945828/Put/seqid=0 2024-11-09T06:15:46,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741883_1059 (size=12151) 2024-11-09T06:15:46,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133006983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:46,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:46,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133006984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133006984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:46,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133006984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:47,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133007003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:47,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133007103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:47,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133007103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:47,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133007314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:47,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133007323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:47,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/772ac1c17672464d82542a9afb84c37c 2024-11-09T06:15:47,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/3207768caa9045dfb8f733fd35b07920 is 50, key is test_row_0/C:col10/1731132945828/Put/seqid=0 2024-11-09T06:15:47,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741884_1060 (size=12151) 2024-11-09T06:15:47,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133007623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:47,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133007633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:47,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/3207768caa9045dfb8f733fd35b07920 2024-11-09T06:15:47,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/0b8a9b102c9b4a0eb3d10d2fc7256a36 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/0b8a9b102c9b4a0eb3d10d2fc7256a36 2024-11-09T06:15:47,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/0b8a9b102c9b4a0eb3d10d2fc7256a36, entries=150, sequenceid=229, filesize=11.9 K 2024-11-09T06:15:47,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/772ac1c17672464d82542a9afb84c37c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/772ac1c17672464d82542a9afb84c37c 2024-11-09T06:15:47,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/772ac1c17672464d82542a9afb84c37c, entries=150, sequenceid=229, filesize=11.9 K 2024-11-09T06:15:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/3207768caa9045dfb8f733fd35b07920 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3207768caa9045dfb8f733fd35b07920 2024-11-09T06:15:47,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3207768caa9045dfb8f733fd35b07920, entries=150, sequenceid=229, filesize=11.9 K 2024-11-09T06:15:47,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 4cec2ccd1a271d540c1f369d10f41e7f in 992ms, sequenceid=229, compaction requested=false 2024-11-09T06:15:47,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:47,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:15:47,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:47,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:47,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:47,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:47,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:47,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:48,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/224eb365bcd64978a99b9e7f28a53a2b is 50, key is test_row_0/A:col10/1731132947993/Put/seqid=0 2024-11-09T06:15:48,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741885_1061 (size=14541) 2024-11-09T06:15:48,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/224eb365bcd64978a99b9e7f28a53a2b 2024-11-09T06:15:48,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133008042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133008042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133008043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/64a19c2d0f894cc0a3c152740e850225 is 50, key is test_row_0/B:col10/1731132947993/Put/seqid=0 2024-11-09T06:15:48,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741886_1062 (size=12151) 2024-11-09T06:15:48,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/64a19c2d0f894cc0a3c152740e850225 2024-11-09T06:15:48,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/2893ca04a2184478ad2bacb7ca491585 is 50, key is test_row_0/C:col10/1731132947993/Put/seqid=0 2024-11-09T06:15:48,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133008132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741887_1063 (size=12151) 2024-11-09T06:15:48,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/2893ca04a2184478ad2bacb7ca491585 2024-11-09T06:15:48,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133008156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/224eb365bcd64978a99b9e7f28a53a2b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/224eb365bcd64978a99b9e7f28a53a2b 2024-11-09T06:15:48,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133008164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133008164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133008163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/224eb365bcd64978a99b9e7f28a53a2b, entries=200, sequenceid=249, filesize=14.2 K 2024-11-09T06:15:48,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/64a19c2d0f894cc0a3c152740e850225 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/64a19c2d0f894cc0a3c152740e850225 2024-11-09T06:15:48,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/64a19c2d0f894cc0a3c152740e850225, entries=150, sequenceid=249, filesize=11.9 K 2024-11-09T06:15:48,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/2893ca04a2184478ad2bacb7ca491585 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2893ca04a2184478ad2bacb7ca491585 2024-11-09T06:15:48,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2893ca04a2184478ad2bacb7ca491585, entries=150, sequenceid=249, filesize=11.9 K 2024-11-09T06:15:48,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for 4cec2ccd1a271d540c1f369d10f41e7f in 232ms, sequenceid=249, compaction requested=true 2024-11-09T06:15:48,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:48,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:15:48,228 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:48,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:48,228 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:48,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:15:48,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:48,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:15:48,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:48,229 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:48,229 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:48,229 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/B is initiating minor compaction (all files) 2024-11-09T06:15:48,229 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/A is initiating minor compaction (all files) 2024-11-09T06:15:48,229 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/A in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:48,229 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/B in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:48,229 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/89e85989c1724b66ad2bcbc6ed0e4ab6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/0b8a9b102c9b4a0eb3d10d2fc7256a36, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/224eb365bcd64978a99b9e7f28a53a2b] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=38.4 K 2024-11-09T06:15:48,229 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/a94e036140ae42ec8f0261257814e1d8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/772ac1c17672464d82542a9afb84c37c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/64a19c2d0f894cc0a3c152740e850225] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.1 K 2024-11-09T06:15:48,230 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e85989c1724b66ad2bcbc6ed0e4ab6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731132945588 2024-11-09T06:15:48,230 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a94e036140ae42ec8f0261257814e1d8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731132945588 2024-11-09T06:15:48,230 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b8a9b102c9b4a0eb3d10d2fc7256a36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731132945828 2024-11-09T06:15:48,230 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 772ac1c17672464d82542a9afb84c37c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731132945828 2024-11-09T06:15:48,231 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 224eb365bcd64978a99b9e7f28a53a2b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731132946922 2024-11-09T06:15:48,231 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 64a19c2d0f894cc0a3c152740e850225, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731132946922 2024-11-09T06:15:48,259 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:48,260 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/df5be43e85af455e96e48d157831e1b4 is 50, key is test_row_0/B:col10/1731132947993/Put/seqid=0 2024-11-09T06:15:48,278 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#A#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:48,279 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/91423c12b6624feea881e77b15089116 is 50, key is test_row_0/A:col10/1731132947993/Put/seqid=0 2024-11-09T06:15:48,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741888_1064 (size=12731) 2024-11-09T06:15:48,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741889_1065 (size=12731) 2024-11-09T06:15:48,313 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/df5be43e85af455e96e48d157831e1b4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/df5be43e85af455e96e48d157831e1b4 2024-11-09T06:15:48,321 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/91423c12b6624feea881e77b15089116 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/91423c12b6624feea881e77b15089116 2024-11-09T06:15:48,331 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/B of 4cec2ccd1a271d540c1f369d10f41e7f into df5be43e85af455e96e48d157831e1b4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:48,331 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:48,331 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/B, priority=13, startTime=1731132948228; duration=0sec 2024-11-09T06:15:48,332 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:48,332 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:B 2024-11-09T06:15:48,335 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:48,337 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/A of 4cec2ccd1a271d540c1f369d10f41e7f into 91423c12b6624feea881e77b15089116(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:48,337 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:48,337 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/A, priority=13, startTime=1731132948227; duration=0sec 2024-11-09T06:15:48,337 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:48,337 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:48,337 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/C is initiating minor compaction (all files) 2024-11-09T06:15:48,337 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:A 2024-11-09T06:15:48,337 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/C in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:48,338 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/107b0887f79d42528f0771059783cf3b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3207768caa9045dfb8f733fd35b07920, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2893ca04a2184478ad2bacb7ca491585] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.1 K 2024-11-09T06:15:48,339 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 107b0887f79d42528f0771059783cf3b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731132945588 2024-11-09T06:15:48,339 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3207768caa9045dfb8f733fd35b07920, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731132945828 2024-11-09T06:15:48,340 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2893ca04a2184478ad2bacb7ca491585, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731132946922 2024-11-09T06:15:48,380 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#C#compaction#53 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:48,384 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/f08402b15fb7490d880fa1dd3c2c2346 is 50, key is test_row_0/C:col10/1731132947993/Put/seqid=0 2024-11-09T06:15:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:48,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T06:15:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:48,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741890_1066 (size=12731) 2024-11-09T06:15:48,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/f9ec0c7fbc5a43d1b2a0905cb0452e0b is 50, key is test_row_0/A:col10/1731132948387/Put/seqid=0 2024-11-09T06:15:48,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133008450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133008459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133008460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741891_1067 (size=12301) 2024-11-09T06:15:48,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/f9ec0c7fbc5a43d1b2a0905cb0452e0b 2024-11-09T06:15:48,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/79048600be8b4f21a7c6e865e2f71682 is 50, key is test_row_0/B:col10/1731132948387/Put/seqid=0 2024-11-09T06:15:48,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741892_1068 (size=12301) 2024-11-09T06:15:48,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/79048600be8b4f21a7c6e865e2f71682 2024-11-09T06:15:48,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/23516da1264d42619fafec2d133182b0 is 50, key is test_row_0/C:col10/1731132948387/Put/seqid=0 2024-11-09T06:15:48,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133008573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133008573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133008577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741893_1069 (size=12301) 2024-11-09T06:15:48,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/23516da1264d42619fafec2d133182b0 2024-11-09T06:15:48,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/f9ec0c7fbc5a43d1b2a0905cb0452e0b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/f9ec0c7fbc5a43d1b2a0905cb0452e0b 2024-11-09T06:15:48,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/f9ec0c7fbc5a43d1b2a0905cb0452e0b, entries=150, sequenceid=268, filesize=12.0 K 2024-11-09T06:15:48,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/79048600be8b4f21a7c6e865e2f71682 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/79048600be8b4f21a7c6e865e2f71682 2024-11-09T06:15:48,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/79048600be8b4f21a7c6e865e2f71682, entries=150, sequenceid=268, filesize=12.0 K 2024-11-09T06:15:48,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/23516da1264d42619fafec2d133182b0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/23516da1264d42619fafec2d133182b0 2024-11-09T06:15:48,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/23516da1264d42619fafec2d133182b0, entries=150, sequenceid=268, filesize=12.0 K 2024-11-09T06:15:48,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4cec2ccd1a271d540c1f369d10f41e7f in 284ms, sequenceid=268, compaction requested=false 2024-11-09T06:15:48,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:48,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T06:15:48,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:48,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:48,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:48,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:48,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:48,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:48,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/2bff311605984de98ad91e3aca23caeb is 50, key is test_row_0/A:col10/1731132948787/Put/seqid=0 2024-11-09T06:15:48,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133008823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133008832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133008834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,868 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/f08402b15fb7490d880fa1dd3c2c2346 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f08402b15fb7490d880fa1dd3c2c2346 2024-11-09T06:15:48,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741894_1070 (size=12301) 2024-11-09T06:15:48,886 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/C of 4cec2ccd1a271d540c1f369d10f41e7f into f08402b15fb7490d880fa1dd3c2c2346(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:48,886 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:48,886 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/C, priority=13, startTime=1731132948228; duration=0sec 2024-11-09T06:15:48,886 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:48,887 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:C 2024-11-09T06:15:48,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133008942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133008943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:48,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133008953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:49,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133009153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:49,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133009154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:49,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:49,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133009155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:49,160 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in 4cec2ccd1a271d540c1f369d10f41e7f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:15:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:49,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133009163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:49,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133009173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:49,179 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in 4cec2ccd1a271d540c1f369d10f41e7f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:15:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:49,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/2bff311605984de98ad91e3aca23caeb 2024-11-09T06:15:49,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/bdd333413c9742d48e18a80ab266c606 is 50, key is test_row_0/B:col10/1731132948787/Put/seqid=0 2024-11-09T06:15:49,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741895_1071 (size=12301) 2024-11-09T06:15:49,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133009466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:49,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133009467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:49,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133009474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:49,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:49,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:49,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/bdd333413c9742d48e18a80ab266c606 2024-11-09T06:15:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T06:15:49,717 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:15:49,718 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:15:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-09T06:15:49,723 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:15:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T06:15:49,724 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:15:49,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:15:49,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/5ec582759efd46f79081a52a13a09dba is 50, key is test_row_0/C:col10/1731132948787/Put/seqid=0 2024-11-09T06:15:49,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741896_1072 (size=12301) 2024-11-09T06:15:49,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/5ec582759efd46f79081a52a13a09dba 2024-11-09T06:15:49,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/2bff311605984de98ad91e3aca23caeb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2bff311605984de98ad91e3aca23caeb 2024-11-09T06:15:49,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2bff311605984de98ad91e3aca23caeb, entries=150, sequenceid=285, filesize=12.0 K 2024-11-09T06:15:49,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/bdd333413c9742d48e18a80ab266c606 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bdd333413c9742d48e18a80ab266c606 2024-11-09T06:15:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T06:15:49,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bdd333413c9742d48e18a80ab266c606, entries=150, sequenceid=285, filesize=12.0 K 2024-11-09T06:15:49,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/5ec582759efd46f79081a52a13a09dba as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/5ec582759efd46f79081a52a13a09dba 2024-11-09T06:15:49,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/5ec582759efd46f79081a52a13a09dba, entries=150, sequenceid=285, filesize=12.0 K 2024-11-09T06:15:49,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4cec2ccd1a271d540c1f369d10f41e7f in 1075ms, sequenceid=285, compaction requested=true 2024-11-09T06:15:49,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:49,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:15:49,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:49,865 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:49,865 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:49,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:15:49,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:49,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:15:49,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:49,869 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:49,869 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:49,870 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/A is initiating minor compaction (all files) 2024-11-09T06:15:49,870 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/A in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:49,870 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/B is initiating minor compaction (all files) 2024-11-09T06:15:49,870 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/B in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:49,870 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/91423c12b6624feea881e77b15089116, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/f9ec0c7fbc5a43d1b2a0905cb0452e0b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2bff311605984de98ad91e3aca23caeb] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.5 K 2024-11-09T06:15:49,871 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/df5be43e85af455e96e48d157831e1b4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/79048600be8b4f21a7c6e865e2f71682, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bdd333413c9742d48e18a80ab266c606] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.5 K 2024-11-09T06:15:49,873 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting df5be43e85af455e96e48d157831e1b4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731132946922 2024-11-09T06:15:49,873 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91423c12b6624feea881e77b15089116, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731132946922 2024-11-09T06:15:49,874 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 79048600be8b4f21a7c6e865e2f71682, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731132948040 2024-11-09T06:15:49,874 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9ec0c7fbc5a43d1b2a0905cb0452e0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731132948040 2024-11-09T06:15:49,876 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting bdd333413c9742d48e18a80ab266c606, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731132948455 2024-11-09T06:15:49,876 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bff311605984de98ad91e3aca23caeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731132948455 2024-11-09T06:15:49,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T06:15:49,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:49,885 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T06:15:49,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:49,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/cfc7f89b6fc24be48c4ad40806af3d3a is 50, key is test_row_0/A:col10/1731132948831/Put/seqid=0 2024-11-09T06:15:49,907 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#A#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:49,908 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/ab5790ee80284024ace481b9128876d9 is 50, key is test_row_0/A:col10/1731132948787/Put/seqid=0 2024-11-09T06:15:49,922 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#B#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:49,923 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/477ebf74610c4385b1906f5e45f85a3e is 50, key is test_row_0/B:col10/1731132948787/Put/seqid=0 2024-11-09T06:15:49,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741897_1073 (size=12301) 2024-11-09T06:15:49,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741899_1075 (size=12983) 2024-11-09T06:15:49,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741898_1074 (size=12983) 2024-11-09T06:15:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:49,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T06:15:50,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133010046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133010046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133010046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133010163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133010163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133010164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T06:15:50,354 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/cfc7f89b6fc24be48c4ad40806af3d3a 2024-11-09T06:15:50,370 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/477ebf74610c4385b1906f5e45f85a3e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/477ebf74610c4385b1906f5e45f85a3e 2024-11-09T06:15:50,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/e3142e9c09ed4ef4bfd5a8d85bbcbb8b is 50, key is test_row_0/B:col10/1731132948831/Put/seqid=0 2024-11-09T06:15:50,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133010384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133010384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133010384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,393 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/B of 4cec2ccd1a271d540c1f369d10f41e7f into 477ebf74610c4385b1906f5e45f85a3e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:50,393 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:50,393 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/B, priority=13, startTime=1731132949865; duration=0sec 2024-11-09T06:15:50,394 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:50,394 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:B 2024-11-09T06:15:50,394 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:50,395 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:50,396 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/C is initiating minor compaction (all files) 2024-11-09T06:15:50,396 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/C in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:50,396 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f08402b15fb7490d880fa1dd3c2c2346, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/23516da1264d42619fafec2d133182b0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/5ec582759efd46f79081a52a13a09dba] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.5 K 2024-11-09T06:15:50,399 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting f08402b15fb7490d880fa1dd3c2c2346, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731132946922 2024-11-09T06:15:50,400 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 23516da1264d42619fafec2d133182b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731132948040 2024-11-09T06:15:50,401 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ec582759efd46f79081a52a13a09dba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731132948455 2024-11-09T06:15:50,402 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/ab5790ee80284024ace481b9128876d9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/ab5790ee80284024ace481b9128876d9 2024-11-09T06:15:50,429 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/A of 4cec2ccd1a271d540c1f369d10f41e7f into ab5790ee80284024ace481b9128876d9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:50,429 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:50,429 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/A, priority=13, startTime=1731132949865; duration=0sec 2024-11-09T06:15:50,429 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:50,429 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:A 2024-11-09T06:15:50,447 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#C#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:50,450 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/138d416b0312472686924ae5d47250b8 is 50, key is test_row_0/C:col10/1731132948787/Put/seqid=0 2024-11-09T06:15:50,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741900_1076 (size=12301) 2024-11-09T06:15:50,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741901_1077 (size=12983) 2024-11-09T06:15:50,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133010693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133010693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133010693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T06:15:50,873 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/e3142e9c09ed4ef4bfd5a8d85bbcbb8b 2024-11-09T06:15:50,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/2c085bf52d524902bd8d4c0c00ab4832 is 50, key is test_row_0/C:col10/1731132948831/Put/seqid=0 2024-11-09T06:15:50,998 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/138d416b0312472686924ae5d47250b8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/138d416b0312472686924ae5d47250b8 2024-11-09T06:15:51,014 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/C of 4cec2ccd1a271d540c1f369d10f41e7f into 138d416b0312472686924ae5d47250b8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:51,014 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:51,014 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/C, priority=13, startTime=1731132949865; duration=0sec 2024-11-09T06:15:51,014 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:51,014 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:C 2024-11-09T06:15:51,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741902_1078 (size=12301) 2024-11-09T06:15:51,016 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/2c085bf52d524902bd8d4c0c00ab4832 2024-11-09T06:15:51,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/cfc7f89b6fc24be48c4ad40806af3d3a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/cfc7f89b6fc24be48c4ad40806af3d3a 2024-11-09T06:15:51,052 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/cfc7f89b6fc24be48c4ad40806af3d3a, entries=150, sequenceid=301, filesize=12.0 K 2024-11-09T06:15:51,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/e3142e9c09ed4ef4bfd5a8d85bbcbb8b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/e3142e9c09ed4ef4bfd5a8d85bbcbb8b 2024-11-09T06:15:51,065 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/e3142e9c09ed4ef4bfd5a8d85bbcbb8b, entries=150, sequenceid=301, filesize=12.0 K 2024-11-09T06:15:51,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/2c085bf52d524902bd8d4c0c00ab4832 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2c085bf52d524902bd8d4c0c00ab4832 2024-11-09T06:15:51,083 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2c085bf52d524902bd8d4c0c00ab4832, entries=150, sequenceid=301, filesize=12.0 K 2024-11-09T06:15:51,085 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4cec2ccd1a271d540c1f369d10f41e7f in 1199ms, sequenceid=301, compaction requested=false 2024-11-09T06:15:51,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:51,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:51,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-09T06:15:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-09T06:15:51,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-09T06:15:51,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3640 sec 2024-11-09T06:15:51,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3760 sec 2024-11-09T06:15:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:51,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T06:15:51,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:51,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:51,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:51,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:51,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:51,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:51,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/5c15360e814544e6b6043d5652d76b99 is 50, key is test_row_0/A:col10/1731132950042/Put/seqid=0 2024-11-09T06:15:51,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741903_1079 (size=14741) 2024-11-09T06:15:51,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133011232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133011238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133011239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133011244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133011246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133011363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133011364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133011364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133011374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133011374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133011583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133011584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133011588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133011584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133011597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/5c15360e814544e6b6043d5652d76b99 2024-11-09T06:15:51,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/7ecfbc68c30243c6892e985338436f69 is 50, key is test_row_0/B:col10/1731132950042/Put/seqid=0 2024-11-09T06:15:51,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741904_1080 (size=12301) 2024-11-09T06:15:51,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/7ecfbc68c30243c6892e985338436f69 2024-11-09T06:15:51,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e5a6f849cd6c4ef39066654829a4b18c is 50, key is test_row_0/C:col10/1731132950042/Put/seqid=0 2024-11-09T06:15:51,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741905_1081 (size=12301) 2024-11-09T06:15:51,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e5a6f849cd6c4ef39066654829a4b18c 2024-11-09T06:15:51,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/5c15360e814544e6b6043d5652d76b99 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5c15360e814544e6b6043d5652d76b99 2024-11-09T06:15:51,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5c15360e814544e6b6043d5652d76b99, entries=200, sequenceid=322, filesize=14.4 K 2024-11-09T06:15:51,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/7ecfbc68c30243c6892e985338436f69 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7ecfbc68c30243c6892e985338436f69 2024-11-09T06:15:51,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7ecfbc68c30243c6892e985338436f69, entries=150, sequenceid=322, filesize=12.0 K 2024-11-09T06:15:51,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e5a6f849cd6c4ef39066654829a4b18c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a6f849cd6c4ef39066654829a4b18c 2024-11-09T06:15:51,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a6f849cd6c4ef39066654829a4b18c, entries=150, sequenceid=322, filesize=12.0 K 2024-11-09T06:15:51,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4cec2ccd1a271d540c1f369d10f41e7f in 640ms, sequenceid=322, compaction requested=true 2024-11-09T06:15:51,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:51,828 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:51,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:15:51,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:51,829 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:51,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:15:51,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:51,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:15:51,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:51,830 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:51,830 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/A is initiating minor compaction (all files) 2024-11-09T06:15:51,830 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/A in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:51,830 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/ab5790ee80284024ace481b9128876d9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/cfc7f89b6fc24be48c4ad40806af3d3a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5c15360e814544e6b6043d5652d76b99] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=39.1 K 2024-11-09T06:15:51,831 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:51,831 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/B is initiating minor compaction (all files) 2024-11-09T06:15:51,831 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/B in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:51,831 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/477ebf74610c4385b1906f5e45f85a3e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/e3142e9c09ed4ef4bfd5a8d85bbcbb8b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7ecfbc68c30243c6892e985338436f69] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.7 K 2024-11-09T06:15:51,833 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 477ebf74610c4385b1906f5e45f85a3e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731132948455 2024-11-09T06:15:51,833 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab5790ee80284024ace481b9128876d9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731132948455 2024-11-09T06:15:51,834 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfc7f89b6fc24be48c4ad40806af3d3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731132948819 2024-11-09T06:15:51,834 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting e3142e9c09ed4ef4bfd5a8d85bbcbb8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731132948819 2024-11-09T06:15:51,835 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ecfbc68c30243c6892e985338436f69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731132950041 2024-11-09T06:15:51,835 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c15360e814544e6b6043d5652d76b99, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731132950034 2024-11-09T06:15:51,867 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:51,868 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/b0af8da11de54b9f889363097c10c0b9 is 50, key is test_row_0/A:col10/1731132950042/Put/seqid=0 2024-11-09T06:15:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T06:15:51,873 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:15:51,874 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:15:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-09T06:15:51,879 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:15:51,880 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T06:15:51,880 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:15:51,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:15:51,881 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/05b97871b04e4473bb33eaecf1dcabd6 is 50, key is test_row_0/B:col10/1731132950042/Put/seqid=0 2024-11-09T06:15:51,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741906_1082 (size=13085) 2024-11-09T06:15:51,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741907_1083 (size=13085) 2024-11-09T06:15:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:51,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T06:15:51,932 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/b0af8da11de54b9f889363097c10c0b9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b0af8da11de54b9f889363097c10c0b9 2024-11-09T06:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:51,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/dbf73646aaff4199ae92dea8f6646712 is 50, key is test_row_0/A:col10/1731132951920/Put/seqid=0 2024-11-09T06:15:51,950 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/A of 4cec2ccd1a271d540c1f369d10f41e7f into b0af8da11de54b9f889363097c10c0b9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:51,950 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:51,950 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/A, priority=13, startTime=1731132951828; duration=0sec 2024-11-09T06:15:51,951 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:51,951 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:A 2024-11-09T06:15:51,951 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:51,953 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:51,953 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/C is initiating minor compaction (all files) 2024-11-09T06:15:51,953 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/C in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:51,953 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/138d416b0312472686924ae5d47250b8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2c085bf52d524902bd8d4c0c00ab4832, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a6f849cd6c4ef39066654829a4b18c] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.7 K 2024-11-09T06:15:51,954 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 138d416b0312472686924ae5d47250b8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731132948455 2024-11-09T06:15:51,955 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c085bf52d524902bd8d4c0c00ab4832, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731132948819 2024-11-09T06:15:51,956 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5a6f849cd6c4ef39066654829a4b18c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731132950041 2024-11-09T06:15:51,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741908_1084 (size=12301) 2024-11-09T06:15:51,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/dbf73646aaff4199ae92dea8f6646712 2024-11-09T06:15:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T06:15:51,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133011968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133011971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,990 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#C#compaction#72 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:51,991 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/f5a7a257aca14780bd80e490463ae81c is 50, key is test_row_0/C:col10/1731132950042/Put/seqid=0 2024-11-09T06:15:51,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133011984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133011987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:51,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:51,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:51,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/d2da6b9b2c244ba68c6a5ae3e6845936 is 50, key is test_row_0/B:col10/1731132951920/Put/seqid=0 2024-11-09T06:15:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133011997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:52,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:52,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741909_1085 (size=13085) 2024-11-09T06:15:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741910_1086 (size=12301) 2024-11-09T06:15:52,060 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/f5a7a257aca14780bd80e490463ae81c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f5a7a257aca14780bd80e490463ae81c 2024-11-09T06:15:52,073 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/C of 4cec2ccd1a271d540c1f369d10f41e7f into f5a7a257aca14780bd80e490463ae81c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:52,073 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:52,074 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/C, priority=13, startTime=1731132951829; duration=0sec 2024-11-09T06:15:52,074 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:52,074 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:C 2024-11-09T06:15:52,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133012093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133012093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133012102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133012103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133012114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T06:15:52,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133012303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133012304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133012323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133012323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133012333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,344 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/05b97871b04e4473bb33eaecf1dcabd6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/05b97871b04e4473bb33eaecf1dcabd6 2024-11-09T06:15:52,358 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/B of 4cec2ccd1a271d540c1f369d10f41e7f into 05b97871b04e4473bb33eaecf1dcabd6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:52,358 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:52,358 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/B, priority=13, startTime=1731132951829; duration=0sec 2024-11-09T06:15:52,358 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:52,358 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:B 2024-11-09T06:15:52,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:52,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/d2da6b9b2c244ba68c6a5ae3e6845936 2024-11-09T06:15:52,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/dae1e09fb8354f8090f11666cad97445 is 50, key is test_row_0/C:col10/1731132951920/Put/seqid=0 2024-11-09T06:15:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T06:15:52,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:52,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:52,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741911_1087 (size=12301) 2024-11-09T06:15:52,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/dae1e09fb8354f8090f11666cad97445 2024-11-09T06:15:52,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/dbf73646aaff4199ae92dea8f6646712 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/dbf73646aaff4199ae92dea8f6646712 2024-11-09T06:15:52,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/dbf73646aaff4199ae92dea8f6646712, entries=150, sequenceid=339, filesize=12.0 K 2024-11-09T06:15:52,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/d2da6b9b2c244ba68c6a5ae3e6845936 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d2da6b9b2c244ba68c6a5ae3e6845936 2024-11-09T06:15:52,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d2da6b9b2c244ba68c6a5ae3e6845936, entries=150, sequenceid=339, filesize=12.0 K 2024-11-09T06:15:52,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/dae1e09fb8354f8090f11666cad97445 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/dae1e09fb8354f8090f11666cad97445 2024-11-09T06:15:52,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/dae1e09fb8354f8090f11666cad97445, entries=150, sequenceid=339, filesize=12.0 K 2024-11-09T06:15:52,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 4cec2ccd1a271d540c1f369d10f41e7f in 648ms, sequenceid=339, compaction requested=false 2024-11-09T06:15:52,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:52,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T06:15:52,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:52,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:52,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:52,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:52,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:52,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:52,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/4899c1a19a714c67b0e76c3ec4ab6a09 is 50, key is test_row_0/A:col10/1731132951965/Put/seqid=0 2024-11-09T06:15:52,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741912_1088 (size=12301) 2024-11-09T06:15:52,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133012650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133012653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133012660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133012662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133012658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133012773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133012773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133012784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133012784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133012784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:52,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:52,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:52,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:52,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133012983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133012982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:52,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133012994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133012997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133012997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:53,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:53,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T06:15:53,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/4899c1a19a714c67b0e76c3ec4ab6a09 2024-11-09T06:15:53,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4afe47b745e248ce9d806e0f62e61273 is 50, key is test_row_0/B:col10/1731132951965/Put/seqid=0 2024-11-09T06:15:53,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741913_1089 (size=12301) 2024-11-09T06:15:53,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:53,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:53,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133013292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133013303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133013303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133013312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:53,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133013322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:53,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:53,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4afe47b745e248ce9d806e0f62e61273 2024-11-09T06:15:53,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/3db0a2f21b594e2f89610bfdc0efc5bb is 50, key is test_row_0/C:col10/1731132951965/Put/seqid=0 2024-11-09T06:15:53,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741914_1090 (size=12301) 2024-11-09T06:15:53,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/3db0a2f21b594e2f89610bfdc0efc5bb 2024-11-09T06:15:53,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/4899c1a19a714c67b0e76c3ec4ab6a09 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/4899c1a19a714c67b0e76c3ec4ab6a09 2024-11-09T06:15:53,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:53,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:53,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:53,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/4899c1a19a714c67b0e76c3ec4ab6a09, entries=150, sequenceid=360, filesize=12.0 K 2024-11-09T06:15:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4afe47b745e248ce9d806e0f62e61273 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4afe47b745e248ce9d806e0f62e61273 2024-11-09T06:15:53,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4afe47b745e248ce9d806e0f62e61273, entries=150, sequenceid=360, filesize=12.0 K 2024-11-09T06:15:53,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/3db0a2f21b594e2f89610bfdc0efc5bb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3db0a2f21b594e2f89610bfdc0efc5bb 2024-11-09T06:15:53,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3db0a2f21b594e2f89610bfdc0efc5bb, entries=150, sequenceid=360, filesize=12.0 K 2024-11-09T06:15:53,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4cec2ccd1a271d540c1f369d10f41e7f in 1058ms, sequenceid=360, compaction requested=true 2024-11-09T06:15:53,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:53,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:15:53,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:53,688 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:53,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:15:53,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:53,688 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:53,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:15:53,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:53,689 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:53,690 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/A is initiating minor compaction (all files) 2024-11-09T06:15:53,690 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/A in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,690 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b0af8da11de54b9f889363097c10c0b9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/dbf73646aaff4199ae92dea8f6646712, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/4899c1a19a714c67b0e76c3ec4ab6a09] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.8 K 2024-11-09T06:15:53,691 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0af8da11de54b9f889363097c10c0b9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731132950041 2024-11-09T06:15:53,691 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:53,691 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbf73646aaff4199ae92dea8f6646712, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731132951920 2024-11-09T06:15:53,691 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/B is initiating minor compaction (all files) 2024-11-09T06:15:53,692 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/B in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,692 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/05b97871b04e4473bb33eaecf1dcabd6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d2da6b9b2c244ba68c6a5ae3e6845936, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4afe47b745e248ce9d806e0f62e61273] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.8 K 2024-11-09T06:15:53,692 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4899c1a19a714c67b0e76c3ec4ab6a09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731132951965 2024-11-09T06:15:53,693 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 05b97871b04e4473bb33eaecf1dcabd6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731132950041 2024-11-09T06:15:53,693 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting d2da6b9b2c244ba68c6a5ae3e6845936, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731132951920 2024-11-09T06:15:53,694 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4afe47b745e248ce9d806e0f62e61273, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731132951965 2024-11-09T06:15:53,717 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:53,718 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/1fba0f518a2f41ddacd931350017a525 is 50, key is test_row_0/A:col10/1731132951965/Put/seqid=0 2024-11-09T06:15:53,724 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:53,725 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/1f7650e50514472d8b544e377376d037 is 50, key is test_row_0/B:col10/1731132951965/Put/seqid=0 2024-11-09T06:15:53,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741915_1091 (size=13187) 2024-11-09T06:15:53,752 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/1fba0f518a2f41ddacd931350017a525 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1fba0f518a2f41ddacd931350017a525 2024-11-09T06:15:53,761 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/A of 4cec2ccd1a271d540c1f369d10f41e7f into 1fba0f518a2f41ddacd931350017a525(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:53,761 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:53,761 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/A, priority=13, startTime=1731132953688; duration=0sec 2024-11-09T06:15:53,761 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:53,761 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:A 2024-11-09T06:15:53,761 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:53,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741916_1092 (size=13187) 2024-11-09T06:15:53,765 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:53,766 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/C is initiating minor compaction (all files) 2024-11-09T06:15:53,766 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/C in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,766 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f5a7a257aca14780bd80e490463ae81c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/dae1e09fb8354f8090f11666cad97445, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3db0a2f21b594e2f89610bfdc0efc5bb] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.8 K 2024-11-09T06:15:53,768 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a7a257aca14780bd80e490463ae81c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731132950041 2024-11-09T06:15:53,769 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting dae1e09fb8354f8090f11666cad97445, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731132951920 2024-11-09T06:15:53,769 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3db0a2f21b594e2f89610bfdc0efc5bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731132951965 2024-11-09T06:15:53,778 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/1f7650e50514472d8b544e377376d037 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/1f7650e50514472d8b544e377376d037 2024-11-09T06:15:53,793 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/B of 4cec2ccd1a271d540c1f369d10f41e7f into 1f7650e50514472d8b544e377376d037(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:53,793 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:53,793 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/B, priority=13, startTime=1731132953688; duration=0sec 2024-11-09T06:15:53,793 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:53,793 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:B 2024-11-09T06:15:53,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T06:15:53,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:53,799 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T06:15:53,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:53,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:53,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:53,800 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#C#compaction#80 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:53,801 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/562424ed1cbb493eb75aa1970111a44d is 50, key is test_row_0/C:col10/1731132951965/Put/seqid=0 2024-11-09T06:15:53,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/91d444e6206f4e6e8e9d64e28b498d4b is 50, key is test_row_0/A:col10/1731132952648/Put/seqid=0 2024-11-09T06:15:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:53,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:53,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741918_1094 (size=12301) 2024-11-09T06:15:53,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741917_1093 (size=13187) 2024-11-09T06:15:53,826 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/562424ed1cbb493eb75aa1970111a44d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/562424ed1cbb493eb75aa1970111a44d 2024-11-09T06:15:53,834 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/C of 4cec2ccd1a271d540c1f369d10f41e7f into 562424ed1cbb493eb75aa1970111a44d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:53,834 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:53,834 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/C, priority=13, startTime=1731132953688; duration=0sec 2024-11-09T06:15:53,835 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:53,835 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:C 2024-11-09T06:15:53,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133013851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133013852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133013854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133013855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133013856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133013962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133013962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133013962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133013962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:53,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133013973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T06:15:54,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133014173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:54,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133014173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:54,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133014173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133014173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:54,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133014183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:54,217 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/91d444e6206f4e6e8e9d64e28b498d4b 2024-11-09T06:15:54,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/bcffffc8932d4ed392c8afe0a8e70039 is 50, key is test_row_0/B:col10/1731132952648/Put/seqid=0 2024-11-09T06:15:54,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741919_1095 (size=12301) 2024-11-09T06:15:54,296 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/bcffffc8932d4ed392c8afe0a8e70039 2024-11-09T06:15:54,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/1502a82eb94b484780be9704350d5c63 is 50, key is test_row_0/C:col10/1731132952648/Put/seqid=0 2024-11-09T06:15:54,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741920_1096 (size=12301) 2024-11-09T06:15:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133014494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133014494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:54,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133014495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:54,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133014504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:54,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133014504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:54,773 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/1502a82eb94b484780be9704350d5c63 2024-11-09T06:15:54,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/91d444e6206f4e6e8e9d64e28b498d4b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/91d444e6206f4e6e8e9d64e28b498d4b 2024-11-09T06:15:54,799 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/91d444e6206f4e6e8e9d64e28b498d4b, entries=150, sequenceid=376, filesize=12.0 K 2024-11-09T06:15:54,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/bcffffc8932d4ed392c8afe0a8e70039 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bcffffc8932d4ed392c8afe0a8e70039 2024-11-09T06:15:54,815 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bcffffc8932d4ed392c8afe0a8e70039, entries=150, sequenceid=376, filesize=12.0 K 2024-11-09T06:15:54,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/1502a82eb94b484780be9704350d5c63 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/1502a82eb94b484780be9704350d5c63 2024-11-09T06:15:54,833 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/1502a82eb94b484780be9704350d5c63, entries=150, sequenceid=376, filesize=12.0 K 2024-11-09T06:15:54,836 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 4cec2ccd1a271d540c1f369d10f41e7f in 1037ms, sequenceid=376, compaction requested=false 2024-11-09T06:15:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:54,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:54,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-09T06:15:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-09T06:15:54,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.9720 sec 2024-11-09T06:15:54,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-09T06:15:54,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9600 sec 2024-11-09T06:15:55,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132953838, max timestamp=1731132955003Num uniques 859; 2024-11-09T06:15:55,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:15:55,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-09T06:15:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:55,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132953838, max timestamp=1731132955003Num uniques 859; 2024-11-09T06:15:55,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:15:55,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:55,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:15:55,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:55,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:15:55,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:55,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:15:55,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/1a863c5215ca4b4e84f0582e42f34664 is 50, key is test_row_0/A:col10/1731132955008/Put/seqid=0 2024-11-09T06:15:55,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741921_1097 (size=12301) 2024-11-09T06:15:55,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133015139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133015137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133015143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133015144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133015156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133015263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133015263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133015263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133015263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133015264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133015472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133015472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133015472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133015473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133015473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/1a863c5215ca4b4e84f0582e42f34664 2024-11-09T06:15:55,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4eb4dbc32a7648198f79637ef49062ec is 50, key is test_row_0/B:col10/1731132955008/Put/seqid=0 2024-11-09T06:15:55,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741922_1098 (size=12301) 2024-11-09T06:15:55,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4eb4dbc32a7648198f79637ef49062ec 2024-11-09T06:15:55,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/cfd4e3e477f949e09be9d64f7c4d6b21 is 50, key is test_row_0/C:col10/1731132955008/Put/seqid=0 2024-11-09T06:15:55,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741923_1099 (size=12301) 2024-11-09T06:15:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133015783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133015783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133015783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133015783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133015784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:55,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/cfd4e3e477f949e09be9d64f7c4d6b21 2024-11-09T06:15:56,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/1a863c5215ca4b4e84f0582e42f34664 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1a863c5215ca4b4e84f0582e42f34664 2024-11-09T06:15:56,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1a863c5215ca4b4e84f0582e42f34664, entries=150, sequenceid=399, filesize=12.0 K 2024-11-09T06:15:56,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/4eb4dbc32a7648198f79637ef49062ec as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4eb4dbc32a7648198f79637ef49062ec 2024-11-09T06:15:56,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4eb4dbc32a7648198f79637ef49062ec, entries=150, sequenceid=399, filesize=12.0 K 2024-11-09T06:15:56,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/cfd4e3e477f949e09be9d64f7c4d6b21 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/cfd4e3e477f949e09be9d64f7c4d6b21 2024-11-09T06:15:56,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/cfd4e3e477f949e09be9d64f7c4d6b21, entries=150, sequenceid=399, filesize=12.0 K 2024-11-09T06:15:56,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 4cec2ccd1a271d540c1f369d10f41e7f in 997ms, sequenceid=399, compaction requested=true 2024-11-09T06:15:56,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:56,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:15:56,032 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:56,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:56,033 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:56,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:15:56,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:56,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:15:56,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:56,035 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:56,035 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/A is initiating minor compaction (all files) 2024-11-09T06:15:56,035 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/A in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:56,035 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1fba0f518a2f41ddacd931350017a525, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/91d444e6206f4e6e8e9d64e28b498d4b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1a863c5215ca4b4e84f0582e42f34664] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.9 K 2024-11-09T06:15:56,038 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:56,038 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/B is initiating minor compaction (all files) 2024-11-09T06:15:56,038 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/B in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:56,039 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/1f7650e50514472d8b544e377376d037, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bcffffc8932d4ed392c8afe0a8e70039, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4eb4dbc32a7648198f79637ef49062ec] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.9 K 2024-11-09T06:15:56,040 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f7650e50514472d8b544e377376d037, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731132951965 2024-11-09T06:15:56,040 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fba0f518a2f41ddacd931350017a525, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731132951965 2024-11-09T06:15:56,040 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting bcffffc8932d4ed392c8afe0a8e70039, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731132952648 2024-11-09T06:15:56,041 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eb4dbc32a7648198f79637ef49062ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731132953838 2024-11-09T06:15:56,041 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91d444e6206f4e6e8e9d64e28b498d4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731132952648 2024-11-09T06:15:56,042 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a863c5215ca4b4e84f0582e42f34664, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731132953838 2024-11-09T06:15:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T06:15:56,043 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:15:56,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:15:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-09T06:15:56,048 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:15:56,050 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:15:56,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:15:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T06:15:56,058 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:56,059 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/022b11d2365545e08b7ccf070fa7b9cd is 50, key is test_row_0/A:col10/1731132955008/Put/seqid=0 2024-11-09T06:15:56,070 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:56,071 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/f8518f4e52a943c29430fe685d50fd4a is 50, key is test_row_0/B:col10/1731132955008/Put/seqid=0 2024-11-09T06:15:56,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741925_1101 (size=13289) 2024-11-09T06:15:56,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741924_1100 (size=13289) 2024-11-09T06:15:56,126 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/f8518f4e52a943c29430fe685d50fd4a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/f8518f4e52a943c29430fe685d50fd4a 2024-11-09T06:15:56,126 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/022b11d2365545e08b7ccf070fa7b9cd as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/022b11d2365545e08b7ccf070fa7b9cd 2024-11-09T06:15:56,138 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/B of 4cec2ccd1a271d540c1f369d10f41e7f into f8518f4e52a943c29430fe685d50fd4a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:56,138 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:56,138 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/B, priority=13, startTime=1731132956032; duration=0sec 2024-11-09T06:15:56,139 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:56,139 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:B 2024-11-09T06:15:56,139 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:56,141 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:56,141 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/C is initiating minor compaction (all files) 2024-11-09T06:15:56,141 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/A of 4cec2ccd1a271d540c1f369d10f41e7f into 022b11d2365545e08b7ccf070fa7b9cd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:56,141 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:56,141 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/C in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:56,141 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/A, priority=13, startTime=1731132956032; duration=0sec 2024-11-09T06:15:56,141 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/562424ed1cbb493eb75aa1970111a44d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/1502a82eb94b484780be9704350d5c63, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/cfd4e3e477f949e09be9d64f7c4d6b21] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=36.9 K 2024-11-09T06:15:56,141 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:56,141 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:A 2024-11-09T06:15:56,141 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 562424ed1cbb493eb75aa1970111a44d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731132951965 2024-11-09T06:15:56,142 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1502a82eb94b484780be9704350d5c63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731132952648 2024-11-09T06:15:56,142 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting cfd4e3e477f949e09be9d64f7c4d6b21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731132953838 2024-11-09T06:15:56,160 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#C#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:56,160 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/b3847a17392242468033ad8593afb658 is 50, key is test_row_0/C:col10/1731132955008/Put/seqid=0 2024-11-09T06:15:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T06:15:56,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741926_1102 (size=13289) 2024-11-09T06:15:56,185 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/b3847a17392242468033ad8593afb658 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/b3847a17392242468033ad8593afb658 2024-11-09T06:15:56,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T06:15:56,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:56,209 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:15:56,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:56,212 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/C of 4cec2ccd1a271d540c1f369d10f41e7f into b3847a17392242468033ad8593afb658(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:56,212 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:56,212 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/C, priority=13, startTime=1731132956033; duration=0sec 2024-11-09T06:15:56,213 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:56,213 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:C 2024-11-09T06:15:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:56,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:56,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:56,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:56,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/5e9259b71bde4c0d996d30c53a045b09 is 50, key is test_row_0/A:col10/1731132955141/Put/seqid=0 2024-11-09T06:15:56,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741927_1103 (size=12301) 2024-11-09T06:15:56,249 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/5e9259b71bde4c0d996d30c53a045b09 2024-11-09T06:15:56,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/b4a24506887a44a3a55b15bb34c05c5e is 50, key is test_row_0/B:col10/1731132955141/Put/seqid=0 2024-11-09T06:15:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:56,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741928_1104 (size=12301) 2024-11-09T06:15:56,331 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/b4a24506887a44a3a55b15bb34c05c5e 2024-11-09T06:15:56,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133016329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133016333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133016338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133016348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133016349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e9f8b762f2564962889476433c2f6e63 is 50, key is test_row_0/C:col10/1731132955141/Put/seqid=0 2024-11-09T06:15:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T06:15:56,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741929_1105 (size=12301) 2024-11-09T06:15:56,408 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e9f8b762f2564962889476433c2f6e63 2024-11-09T06:15:56,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/5e9259b71bde4c0d996d30c53a045b09 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5e9259b71bde4c0d996d30c53a045b09 2024-11-09T06:15:56,428 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5e9259b71bde4c0d996d30c53a045b09, entries=150, sequenceid=421, filesize=12.0 K 2024-11-09T06:15:56,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/b4a24506887a44a3a55b15bb34c05c5e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b4a24506887a44a3a55b15bb34c05c5e 2024-11-09T06:15:56,436 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b4a24506887a44a3a55b15bb34c05c5e, entries=150, sequenceid=421, filesize=12.0 K 2024-11-09T06:15:56,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/e9f8b762f2564962889476433c2f6e63 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e9f8b762f2564962889476433c2f6e63 2024-11-09T06:15:56,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133016453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133016454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133016454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,459 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e9f8b762f2564962889476433c2f6e63, entries=150, sequenceid=421, filesize=12.0 K 2024-11-09T06:15:56,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,462 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4cec2ccd1a271d540c1f369d10f41e7f in 254ms, sequenceid=421, compaction requested=false 2024-11-09T06:15:56,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:56,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:56,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-09T06:15:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-09T06:15:56,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:15:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:56,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-09T06:15:56,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 415 msec 2024-11-09T06:15:56,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 427 msec 2024-11-09T06:15:56,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/478c53c20dc74c1487c90fd570228f8c is 50, key is test_row_0/A:col10/1731132956335/Put/seqid=0 2024-11-09T06:15:56,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741930_1106 (size=12297) 2024-11-09T06:15:56,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133016528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133016541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133016653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133016656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133016663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133016673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133016673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T06:15:56,683 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:15:56,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:15:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-09T06:15:56,691 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-09T06:15:56,691 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:15:56,695 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-09T06:15:56,695 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:15:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T06:15:56,697 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-09T06:15:56,698 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:15:56,698 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:56,699 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:56,700 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:56,700 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:56,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:15:56,700 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:56,700 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:56,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:15:56,701 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-09T06:15:56,701 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:15:56,701 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:56,701 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:56,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:15:56,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:15:56,702 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:15:56,704 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:15:56,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T06:15:56,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T06:15:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:56,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:56,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:56,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133016863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133016872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/478c53c20dc74c1487c90fd570228f8c 2024-11-09T06:15:56,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/b3740cdce6ff469e82880d28eb2e6a6d is 50, key is test_row_0/B:col10/1731132956335/Put/seqid=0 2024-11-09T06:15:56,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741931_1107 (size=9857) 2024-11-09T06:15:56,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133016983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133016983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133016983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:57,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T06:15:57,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:57,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T06:15:57,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T06:15:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133017173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:57,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133017183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:57,298 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T06:15:57,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T06:15:57,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:57,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T06:15:57,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/b3740cdce6ff469e82880d28eb2e6a6d 2024-11-09T06:15:57,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/3883a2bb7c3b4751a7409acafb59eddf is 50, key is test_row_0/C:col10/1731132956335/Put/seqid=0 2024-11-09T06:15:57,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741932_1108 (size=9857) 2024-11-09T06:15:57,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T06:15:57,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:57,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:57,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:57,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34258 deadline: 1731133017493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34256 deadline: 1731133017493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1731133017493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:57,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T06:15:57,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:57,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34270 deadline: 1731133017683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:57,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:15:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34216 deadline: 1731133017693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:15:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:15:57,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/3883a2bb7c3b4751a7409acafb59eddf 2024-11-09T06:15:57,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/478c53c20dc74c1487c90fd570228f8c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/478c53c20dc74c1487c90fd570228f8c 2024-11-09T06:15:57,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/478c53c20dc74c1487c90fd570228f8c, entries=150, sequenceid=435, filesize=12.0 K 2024-11-09T06:15:57,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/b3740cdce6ff469e82880d28eb2e6a6d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b3740cdce6ff469e82880d28eb2e6a6d 2024-11-09T06:15:57,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T06:15:57,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:57,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:15:57,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b3740cdce6ff469e82880d28eb2e6a6d, entries=100, sequenceid=435, filesize=9.6 K 2024-11-09T06:15:57,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/3883a2bb7c3b4751a7409acafb59eddf as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3883a2bb7c3b4751a7409acafb59eddf 2024-11-09T06:15:57,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3883a2bb7c3b4751a7409acafb59eddf, entries=100, sequenceid=435, filesize=9.6 K 2024-11-09T06:15:57,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4cec2ccd1a271d540c1f369d10f41e7f in 1327ms, sequenceid=435, compaction requested=true 2024-11-09T06:15:57,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:57,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:15:57,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:57,793 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:57,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:15:57,793 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:57,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:57,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4cec2ccd1a271d540c1f369d10f41e7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:15:57,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:57,794 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:57,794 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37887 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:57,794 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/B is initiating minor compaction (all files) 2024-11-09T06:15:57,794 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/A is initiating minor compaction (all files) 2024-11-09T06:15:57,795 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/B in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,795 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/A in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,795 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/f8518f4e52a943c29430fe685d50fd4a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b4a24506887a44a3a55b15bb34c05c5e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b3740cdce6ff469e82880d28eb2e6a6d] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=34.6 K 2024-11-09T06:15:57,795 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/022b11d2365545e08b7ccf070fa7b9cd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5e9259b71bde4c0d996d30c53a045b09, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/478c53c20dc74c1487c90fd570228f8c] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=37.0 K 2024-11-09T06:15:57,795 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting f8518f4e52a943c29430fe685d50fd4a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731132953838 2024-11-09T06:15:57,795 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 022b11d2365545e08b7ccf070fa7b9cd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731132953838 2024-11-09T06:15:57,795 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e9259b71bde4c0d996d30c53a045b09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731132955116 2024-11-09T06:15:57,795 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b4a24506887a44a3a55b15bb34c05c5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731132955116 2024-11-09T06:15:57,796 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 478c53c20dc74c1487c90fd570228f8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731132956335 2024-11-09T06:15:57,796 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b3740cdce6ff469e82880d28eb2e6a6d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731132956335 2024-11-09T06:15:57,803 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:57,804 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#B#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:57,804 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/d8e65fad35fa4e42a990ca4689161280 is 50, key is test_row_0/A:col10/1731132956335/Put/seqid=0 2024-11-09T06:15:57,804 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/d317b8a9ea0647e9b609d27319688cb7 is 50, key is test_row_0/B:col10/1731132956335/Put/seqid=0 2024-11-09T06:15:57,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741933_1109 (size=13391) 2024-11-09T06:15:57,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741934_1110 (size=13391) 2024-11-09T06:15:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T06:15:57,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T06:15:57,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:57,934 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T06:15:57,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:57,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:57,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:57,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:57,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:57,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:57,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/b9798293f06647a09959977e3e2fc3e0 is 50, key is test_row_0/A:col10/1731132956538/Put/seqid=0 2024-11-09T06:15:57,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741935_1111 (size=12301) 2024-11-09T06:15:58,216 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/d8e65fad35fa4e42a990ca4689161280 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d8e65fad35fa4e42a990ca4689161280 2024-11-09T06:15:58,216 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/d317b8a9ea0647e9b609d27319688cb7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d317b8a9ea0647e9b609d27319688cb7 2024-11-09T06:15:58,222 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/A of 4cec2ccd1a271d540c1f369d10f41e7f into d8e65fad35fa4e42a990ca4689161280(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:58,222 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/B of 4cec2ccd1a271d540c1f369d10f41e7f into d317b8a9ea0647e9b609d27319688cb7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:58,222 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:58,222 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:58,222 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/A, priority=13, startTime=1731132957793; duration=0sec 2024-11-09T06:15:58,222 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/B, priority=13, startTime=1731132957793; duration=0sec 2024-11-09T06:15:58,223 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:15:58,223 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:A 2024-11-09T06:15:58,223 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:58,223 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:B 2024-11-09T06:15:58,223 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:15:58,224 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:15:58,224 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 4cec2ccd1a271d540c1f369d10f41e7f/C is initiating minor compaction (all files) 2024-11-09T06:15:58,224 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4cec2ccd1a271d540c1f369d10f41e7f/C in TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:58,224 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/b3847a17392242468033ad8593afb658, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e9f8b762f2564962889476433c2f6e63, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3883a2bb7c3b4751a7409acafb59eddf] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp, totalSize=34.6 K 2024-11-09T06:15:58,225 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3847a17392242468033ad8593afb658, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731132953838 2024-11-09T06:15:58,225 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9f8b762f2564962889476433c2f6e63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731132955116 2024-11-09T06:15:58,225 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3883a2bb7c3b4751a7409acafb59eddf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731132956335 2024-11-09T06:15:58,234 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cec2ccd1a271d540c1f369d10f41e7f#C#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:15:58,235 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/6482e2fb08ec42d2a80ebfac7eb2db68 is 50, key is test_row_0/C:col10/1731132956335/Put/seqid=0 2024-11-09T06:15:58,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741936_1112 (size=13391) 2024-11-09T06:15:58,347 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/b9798293f06647a09959977e3e2fc3e0 2024-11-09T06:15:58,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/0be6a11468cf46418f0d0dcda17dd7bb is 50, key is test_row_0/B:col10/1731132956538/Put/seqid=0 2024-11-09T06:15:58,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741937_1113 (size=12301) 2024-11-09T06:15:58,371 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/0be6a11468cf46418f0d0dcda17dd7bb 2024-11-09T06:15:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/c7c30f6791164748845520cd5ef3e545 is 50, key is test_row_0/C:col10/1731132956538/Put/seqid=0 2024-11-09T06:15:58,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741938_1114 (size=12301) 2024-11-09T06:15:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:58,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. as already flushing 2024-11-09T06:15:58,504 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-09T06:15:58,504 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-09T06:15:58,504 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:15:58,504 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:15:58,504 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,504 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,505 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,505 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:15:58,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:15:58,515 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-09T06:15:58,515 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:15:58,515 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,515 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:15:58,645 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/6482e2fb08ec42d2a80ebfac7eb2db68 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/6482e2fb08ec42d2a80ebfac7eb2db68 2024-11-09T06:15:58,651 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4cec2ccd1a271d540c1f369d10f41e7f/C of 4cec2ccd1a271d540c1f369d10f41e7f into 6482e2fb08ec42d2a80ebfac7eb2db68(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:15:58,651 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:58,651 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., storeName=4cec2ccd1a271d540c1f369d10f41e7f/C, priority=13, startTime=1731132957793; duration=0sec 2024-11-09T06:15:58,651 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:15:58,651 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cec2ccd1a271d540c1f369d10f41e7f:C 2024-11-09T06:15:58,694 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-09T06:15:58,694 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:15:58,694 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,694 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:15:58,715 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-09T06:15:58,715 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:15:58,715 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,715 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:15:58,788 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/c7c30f6791164748845520cd5ef3e545 2024-11-09T06:15:58,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/b9798293f06647a09959977e3e2fc3e0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b9798293f06647a09959977e3e2fc3e0 2024-11-09T06:15:58,800 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b9798293f06647a09959977e3e2fc3e0, entries=150, sequenceid=454, filesize=12.0 K 2024-11-09T06:15:58,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/0be6a11468cf46418f0d0dcda17dd7bb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/0be6a11468cf46418f0d0dcda17dd7bb 2024-11-09T06:15:58,807 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/0be6a11468cf46418f0d0dcda17dd7bb, entries=150, sequenceid=454, filesize=12.0 K 2024-11-09T06:15:58,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/c7c30f6791164748845520cd5ef3e545 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c7c30f6791164748845520cd5ef3e545 2024-11-09T06:15:58,814 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c7c30f6791164748845520cd5ef3e545, entries=150, sequenceid=454, filesize=12.0 K 2024-11-09T06:15:58,815 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=33.54 KB/34350 for 4cec2ccd1a271d540c1f369d10f41e7f in 881ms, sequenceid=454, compaction requested=false 2024-11-09T06:15:58,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 4cec2ccd1a271d540c1f369d10f41e7f: 2024-11-09T06:15:58,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:58,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-09T06:15:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-09T06:15:58,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-09T06:15:58,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1130 sec 2024-11-09T06:15:58,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.1350 sec 2024-11-09T06:15:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T06:15:58,843 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:15:58,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T06:15:58,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-09T06:15:58,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-09T06:15:58,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-09T06:15:58,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-09T06:15:58,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-09T06:15:58,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T06:15:58,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1736 2024-11-09T06:15:58,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1752 2024-11-09T06:15:58,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T06:15:58,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 838 2024-11-09T06:15:58,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2498 rows 2024-11-09T06:15:58,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 853 2024-11-09T06:15:58,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2539 rows 2024-11-09T06:15:58,844 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T06:15:58,844 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T06:15:58,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:15:58,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:15:58,850 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T06:15:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T06:15:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T06:15:58,859 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731132958858"}]},"ts":"1731132958858"} 2024-11-09T06:15:58,861 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T06:15:58,861 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T06:15:58,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T06:15:58,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4cec2ccd1a271d540c1f369d10f41e7f, UNASSIGN}] 2024-11-09T06:15:58,868 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4cec2ccd1a271d540c1f369d10f41e7f, UNASSIGN 2024-11-09T06:15:58,870 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4cec2ccd1a271d540c1f369d10f41e7f, regionState=CLOSING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:15:58,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4cec2ccd1a271d540c1f369d10f41e7f, UNASSIGN because future has completed 2024-11-09T06:15:58,873 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T06:15:58,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:15:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T06:15:59,031 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:15:59,031 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T06:15:59,032 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 4cec2ccd1a271d540c1f369d10f41e7f, disabling compactions & flushes 2024-11-09T06:15:59,032 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:59,032 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:59,032 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. after waiting 0 ms 2024-11-09T06:15:59,032 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:15:59,033 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 4cec2ccd1a271d540c1f369d10f41e7f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T06:15:59,033 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=A 2024-11-09T06:15:59,033 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:59,033 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=B 2024-11-09T06:15:59,033 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:59,033 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4cec2ccd1a271d540c1f369d10f41e7f, store=C 2024-11-09T06:15:59,033 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:15:59,038 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/faa6280f9a364feda2084481dfed9699 is 50, key is test_row_0/A:col10/1731132958513/Put/seqid=0 2024-11-09T06:15:59,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741939_1115 (size=12301) 2024-11-09T06:15:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T06:15:59,453 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/faa6280f9a364feda2084481dfed9699 2024-11-09T06:15:59,464 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/a5f61daf1590482e9bf82f2003ae502d is 50, key is test_row_0/B:col10/1731132958513/Put/seqid=0 2024-11-09T06:15:59,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741940_1116 (size=12301) 2024-11-09T06:15:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T06:15:59,869 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/a5f61daf1590482e9bf82f2003ae502d 2024-11-09T06:15:59,886 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/fbe6487af46d4c00bfb32d41686fdfd3 is 50, key is test_row_0/C:col10/1731132958513/Put/seqid=0 2024-11-09T06:15:59,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741941_1117 (size=12301) 2024-11-09T06:15:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T06:16:00,293 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/fbe6487af46d4c00bfb32d41686fdfd3 2024-11-09T06:16:00,300 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/A/faa6280f9a364feda2084481dfed9699 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/faa6280f9a364feda2084481dfed9699 2024-11-09T06:16:00,308 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/faa6280f9a364feda2084481dfed9699, entries=150, sequenceid=465, filesize=12.0 K 2024-11-09T06:16:00,309 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/B/a5f61daf1590482e9bf82f2003ae502d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/a5f61daf1590482e9bf82f2003ae502d 2024-11-09T06:16:00,317 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/a5f61daf1590482e9bf82f2003ae502d, entries=150, sequenceid=465, filesize=12.0 K 2024-11-09T06:16:00,319 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/.tmp/C/fbe6487af46d4c00bfb32d41686fdfd3 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/fbe6487af46d4c00bfb32d41686fdfd3 2024-11-09T06:16:00,328 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/fbe6487af46d4c00bfb32d41686fdfd3, entries=150, sequenceid=465, filesize=12.0 K 2024-11-09T06:16:00,329 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4cec2ccd1a271d540c1f369d10f41e7f in 1297ms, sequenceid=465, compaction requested=true 2024-11-09T06:16:00,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/10a640b45e1843bdb759b84f2dff96e3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2307cc57e7f349efb3b0ee7eba0e4b23, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/bc9bb7953f30483c93b84ece66ff26c9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/a0cad2c2bd6a4285b3fc56e43701e094, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/7bead93fd3d84f55b6add64a389eff0f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/48445f1ad89c4a319e348996ba7ee2f6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d386b838052949368607e251de4b6413, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d0ba71543ee94bf09e2806a13cf65dee, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/9e1574b7b69542e995f70817894194bc, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/076d1959c98944d2bd5758080f42c90b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b5a2ac599be943429f1c7b860e45caf7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/c6f157f3c80a4ad8aee78cf70ba7fa4e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/70de94b114794474a34dbff5e2a5827a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1c5825c9b6fd455191e7a17941051a74, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/89e85989c1724b66ad2bcbc6ed0e4ab6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/0b8a9b102c9b4a0eb3d10d2fc7256a36, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/224eb365bcd64978a99b9e7f28a53a2b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/91423c12b6624feea881e77b15089116, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/f9ec0c7fbc5a43d1b2a0905cb0452e0b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/ab5790ee80284024ace481b9128876d9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2bff311605984de98ad91e3aca23caeb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/cfc7f89b6fc24be48c4ad40806af3d3a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5c15360e814544e6b6043d5652d76b99, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b0af8da11de54b9f889363097c10c0b9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/dbf73646aaff4199ae92dea8f6646712, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1fba0f518a2f41ddacd931350017a525, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/4899c1a19a714c67b0e76c3ec4ab6a09, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/91d444e6206f4e6e8e9d64e28b498d4b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/022b11d2365545e08b7ccf070fa7b9cd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1a863c5215ca4b4e84f0582e42f34664, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5e9259b71bde4c0d996d30c53a045b09, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/478c53c20dc74c1487c90fd570228f8c] to archive 2024-11-09T06:16:00,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:16:00,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/10a640b45e1843bdb759b84f2dff96e3 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/10a640b45e1843bdb759b84f2dff96e3 2024-11-09T06:16:00,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2307cc57e7f349efb3b0ee7eba0e4b23 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2307cc57e7f349efb3b0ee7eba0e4b23 2024-11-09T06:16:00,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/bc9bb7953f30483c93b84ece66ff26c9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/bc9bb7953f30483c93b84ece66ff26c9 2024-11-09T06:16:00,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/a0cad2c2bd6a4285b3fc56e43701e094 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/a0cad2c2bd6a4285b3fc56e43701e094 2024-11-09T06:16:00,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/7bead93fd3d84f55b6add64a389eff0f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/7bead93fd3d84f55b6add64a389eff0f 2024-11-09T06:16:00,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/48445f1ad89c4a319e348996ba7ee2f6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/48445f1ad89c4a319e348996ba7ee2f6 2024-11-09T06:16:00,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d386b838052949368607e251de4b6413 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d386b838052949368607e251de4b6413 2024-11-09T06:16:00,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d0ba71543ee94bf09e2806a13cf65dee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d0ba71543ee94bf09e2806a13cf65dee 2024-11-09T06:16:00,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/9e1574b7b69542e995f70817894194bc to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/9e1574b7b69542e995f70817894194bc 2024-11-09T06:16:00,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/076d1959c98944d2bd5758080f42c90b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/076d1959c98944d2bd5758080f42c90b 2024-11-09T06:16:00,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b5a2ac599be943429f1c7b860e45caf7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b5a2ac599be943429f1c7b860e45caf7 2024-11-09T06:16:00,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/c6f157f3c80a4ad8aee78cf70ba7fa4e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/c6f157f3c80a4ad8aee78cf70ba7fa4e 2024-11-09T06:16:00,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/70de94b114794474a34dbff5e2a5827a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/70de94b114794474a34dbff5e2a5827a 2024-11-09T06:16:00,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1c5825c9b6fd455191e7a17941051a74 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1c5825c9b6fd455191e7a17941051a74 2024-11-09T06:16:00,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/89e85989c1724b66ad2bcbc6ed0e4ab6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/89e85989c1724b66ad2bcbc6ed0e4ab6 2024-11-09T06:16:00,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/0b8a9b102c9b4a0eb3d10d2fc7256a36 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/0b8a9b102c9b4a0eb3d10d2fc7256a36 2024-11-09T06:16:00,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/224eb365bcd64978a99b9e7f28a53a2b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/224eb365bcd64978a99b9e7f28a53a2b 2024-11-09T06:16:00,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/91423c12b6624feea881e77b15089116 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/91423c12b6624feea881e77b15089116 2024-11-09T06:16:00,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/f9ec0c7fbc5a43d1b2a0905cb0452e0b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/f9ec0c7fbc5a43d1b2a0905cb0452e0b 2024-11-09T06:16:00,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/ab5790ee80284024ace481b9128876d9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/ab5790ee80284024ace481b9128876d9 2024-11-09T06:16:00,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2bff311605984de98ad91e3aca23caeb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/2bff311605984de98ad91e3aca23caeb 2024-11-09T06:16:00,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/cfc7f89b6fc24be48c4ad40806af3d3a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/cfc7f89b6fc24be48c4ad40806af3d3a 2024-11-09T06:16:00,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5c15360e814544e6b6043d5652d76b99 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5c15360e814544e6b6043d5652d76b99 2024-11-09T06:16:00,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b0af8da11de54b9f889363097c10c0b9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b0af8da11de54b9f889363097c10c0b9 2024-11-09T06:16:00,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/dbf73646aaff4199ae92dea8f6646712 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/dbf73646aaff4199ae92dea8f6646712 2024-11-09T06:16:00,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1fba0f518a2f41ddacd931350017a525 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1fba0f518a2f41ddacd931350017a525 2024-11-09T06:16:00,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/4899c1a19a714c67b0e76c3ec4ab6a09 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/4899c1a19a714c67b0e76c3ec4ab6a09 2024-11-09T06:16:00,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/91d444e6206f4e6e8e9d64e28b498d4b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/91d444e6206f4e6e8e9d64e28b498d4b 2024-11-09T06:16:00,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/022b11d2365545e08b7ccf070fa7b9cd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/022b11d2365545e08b7ccf070fa7b9cd 2024-11-09T06:16:00,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1a863c5215ca4b4e84f0582e42f34664 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/1a863c5215ca4b4e84f0582e42f34664 2024-11-09T06:16:00,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5e9259b71bde4c0d996d30c53a045b09 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/5e9259b71bde4c0d996d30c53a045b09 2024-11-09T06:16:00,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/478c53c20dc74c1487c90fd570228f8c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/478c53c20dc74c1487c90fd570228f8c 2024-11-09T06:16:00,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/f251190d659344c9983584f6fe96fc8c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7fa26b975ab6424ba4e9167d07a9bf7b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/66019a6376674dfba3f8c102ebf9e6e6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/9e6d90e9db654d8e82882438966a920e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/5bf84afedabe4e2fa634fd642cfa16ce, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/1d94775cc07245478ea60e85a05108ea, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/61182a4c7750450b9f7a9243a9ac93bd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/956f763c38cf4d898304cb50a21662ea, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/9480dfe0e6f54868a662d9b4e577a764, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d3df944ed84a463ca3630620f334fc72, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/32744b241509465ead823e7510b7add3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/8046f82778c140ccb255ee42082d0945, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4a8e06931eeb48d19719d2cc7a7cab0d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/a94e036140ae42ec8f0261257814e1d8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4d881a3c916241878c6ac395b7bcd37a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/772ac1c17672464d82542a9afb84c37c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/df5be43e85af455e96e48d157831e1b4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/64a19c2d0f894cc0a3c152740e850225, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/79048600be8b4f21a7c6e865e2f71682, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/477ebf74610c4385b1906f5e45f85a3e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bdd333413c9742d48e18a80ab266c606, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/e3142e9c09ed4ef4bfd5a8d85bbcbb8b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/05b97871b04e4473bb33eaecf1dcabd6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7ecfbc68c30243c6892e985338436f69, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d2da6b9b2c244ba68c6a5ae3e6845936, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/1f7650e50514472d8b544e377376d037, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4afe47b745e248ce9d806e0f62e61273, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bcffffc8932d4ed392c8afe0a8e70039, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/f8518f4e52a943c29430fe685d50fd4a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4eb4dbc32a7648198f79637ef49062ec, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b4a24506887a44a3a55b15bb34c05c5e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b3740cdce6ff469e82880d28eb2e6a6d] to archive 2024-11-09T06:16:00,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:16:00,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/f251190d659344c9983584f6fe96fc8c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/f251190d659344c9983584f6fe96fc8c 2024-11-09T06:16:00,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7fa26b975ab6424ba4e9167d07a9bf7b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7fa26b975ab6424ba4e9167d07a9bf7b 2024-11-09T06:16:00,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/66019a6376674dfba3f8c102ebf9e6e6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/66019a6376674dfba3f8c102ebf9e6e6 2024-11-09T06:16:00,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/9e6d90e9db654d8e82882438966a920e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/9e6d90e9db654d8e82882438966a920e 2024-11-09T06:16:00,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/5bf84afedabe4e2fa634fd642cfa16ce to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/5bf84afedabe4e2fa634fd642cfa16ce 2024-11-09T06:16:00,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/1d94775cc07245478ea60e85a05108ea to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/1d94775cc07245478ea60e85a05108ea 2024-11-09T06:16:00,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/61182a4c7750450b9f7a9243a9ac93bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/61182a4c7750450b9f7a9243a9ac93bd 2024-11-09T06:16:00,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/956f763c38cf4d898304cb50a21662ea to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/956f763c38cf4d898304cb50a21662ea 2024-11-09T06:16:00,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/9480dfe0e6f54868a662d9b4e577a764 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/9480dfe0e6f54868a662d9b4e577a764 2024-11-09T06:16:00,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d3df944ed84a463ca3630620f334fc72 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d3df944ed84a463ca3630620f334fc72 2024-11-09T06:16:00,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/32744b241509465ead823e7510b7add3 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/32744b241509465ead823e7510b7add3 2024-11-09T06:16:00,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/8046f82778c140ccb255ee42082d0945 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/8046f82778c140ccb255ee42082d0945 2024-11-09T06:16:00,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4a8e06931eeb48d19719d2cc7a7cab0d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4a8e06931eeb48d19719d2cc7a7cab0d 2024-11-09T06:16:00,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/a94e036140ae42ec8f0261257814e1d8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/a94e036140ae42ec8f0261257814e1d8 2024-11-09T06:16:00,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4d881a3c916241878c6ac395b7bcd37a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4d881a3c916241878c6ac395b7bcd37a 2024-11-09T06:16:00,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/772ac1c17672464d82542a9afb84c37c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/772ac1c17672464d82542a9afb84c37c 2024-11-09T06:16:00,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/df5be43e85af455e96e48d157831e1b4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/df5be43e85af455e96e48d157831e1b4 2024-11-09T06:16:00,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/64a19c2d0f894cc0a3c152740e850225 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/64a19c2d0f894cc0a3c152740e850225 2024-11-09T06:16:00,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/79048600be8b4f21a7c6e865e2f71682 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/79048600be8b4f21a7c6e865e2f71682 2024-11-09T06:16:00,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/477ebf74610c4385b1906f5e45f85a3e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/477ebf74610c4385b1906f5e45f85a3e 2024-11-09T06:16:00,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bdd333413c9742d48e18a80ab266c606 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bdd333413c9742d48e18a80ab266c606 2024-11-09T06:16:00,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/e3142e9c09ed4ef4bfd5a8d85bbcbb8b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/e3142e9c09ed4ef4bfd5a8d85bbcbb8b 2024-11-09T06:16:00,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/05b97871b04e4473bb33eaecf1dcabd6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/05b97871b04e4473bb33eaecf1dcabd6 2024-11-09T06:16:00,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7ecfbc68c30243c6892e985338436f69 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/7ecfbc68c30243c6892e985338436f69 2024-11-09T06:16:00,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d2da6b9b2c244ba68c6a5ae3e6845936 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d2da6b9b2c244ba68c6a5ae3e6845936 2024-11-09T06:16:00,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/1f7650e50514472d8b544e377376d037 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/1f7650e50514472d8b544e377376d037 2024-11-09T06:16:00,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4afe47b745e248ce9d806e0f62e61273 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4afe47b745e248ce9d806e0f62e61273 2024-11-09T06:16:00,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bcffffc8932d4ed392c8afe0a8e70039 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/bcffffc8932d4ed392c8afe0a8e70039 2024-11-09T06:16:00,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/f8518f4e52a943c29430fe685d50fd4a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/f8518f4e52a943c29430fe685d50fd4a 2024-11-09T06:16:00,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4eb4dbc32a7648198f79637ef49062ec to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/4eb4dbc32a7648198f79637ef49062ec 2024-11-09T06:16:00,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b4a24506887a44a3a55b15bb34c05c5e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b4a24506887a44a3a55b15bb34c05c5e 2024-11-09T06:16:00,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b3740cdce6ff469e82880d28eb2e6a6d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/b3740cdce6ff469e82880d28eb2e6a6d 2024-11-09T06:16:00,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/a846b8b826644140a1716b58fd6938f7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c94b7164c0074958a197b0314ea502fe, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/572fb078844747bd9d216658a4001182, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/557a13bedfdc4beb9339b7df26db8c92, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/ff069f01e205472e9ca8573817b80b8f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f8a06a719f274889ac66f3150c2eab5c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e2169ea7bf21469488abf508c6529b6b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2d4239b7e6494f30891ae6676dcebf75, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/eb6cea8d2ac04539b3d43f59d7131d06, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/7682e22a57274f8694fe22109a6a3cb9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e24d31a2abfa45a591359951346601c4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/73e4a19b9af14140adcbf4a699504f4c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a69727a94b49ebb3deac2b7fa6ad09, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/107b0887f79d42528f0771059783cf3b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c7dbc3d2fb4345f080b690e6dba261c6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3207768caa9045dfb8f733fd35b07920, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f08402b15fb7490d880fa1dd3c2c2346, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2893ca04a2184478ad2bacb7ca491585, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/23516da1264d42619fafec2d133182b0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/138d416b0312472686924ae5d47250b8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/5ec582759efd46f79081a52a13a09dba, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2c085bf52d524902bd8d4c0c00ab4832, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f5a7a257aca14780bd80e490463ae81c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a6f849cd6c4ef39066654829a4b18c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/dae1e09fb8354f8090f11666cad97445, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/562424ed1cbb493eb75aa1970111a44d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3db0a2f21b594e2f89610bfdc0efc5bb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/1502a82eb94b484780be9704350d5c63, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/b3847a17392242468033ad8593afb658, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/cfd4e3e477f949e09be9d64f7c4d6b21, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e9f8b762f2564962889476433c2f6e63, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3883a2bb7c3b4751a7409acafb59eddf] to archive 2024-11-09T06:16:00,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:16:00,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/a846b8b826644140a1716b58fd6938f7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/a846b8b826644140a1716b58fd6938f7 2024-11-09T06:16:00,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c94b7164c0074958a197b0314ea502fe to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c94b7164c0074958a197b0314ea502fe 2024-11-09T06:16:00,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/572fb078844747bd9d216658a4001182 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/572fb078844747bd9d216658a4001182 2024-11-09T06:16:00,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/557a13bedfdc4beb9339b7df26db8c92 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/557a13bedfdc4beb9339b7df26db8c92 2024-11-09T06:16:00,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/ff069f01e205472e9ca8573817b80b8f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/ff069f01e205472e9ca8573817b80b8f 2024-11-09T06:16:00,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f8a06a719f274889ac66f3150c2eab5c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f8a06a719f274889ac66f3150c2eab5c 2024-11-09T06:16:00,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e2169ea7bf21469488abf508c6529b6b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e2169ea7bf21469488abf508c6529b6b 2024-11-09T06:16:00,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2d4239b7e6494f30891ae6676dcebf75 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2d4239b7e6494f30891ae6676dcebf75 2024-11-09T06:16:00,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/eb6cea8d2ac04539b3d43f59d7131d06 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/eb6cea8d2ac04539b3d43f59d7131d06 2024-11-09T06:16:00,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/7682e22a57274f8694fe22109a6a3cb9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/7682e22a57274f8694fe22109a6a3cb9 2024-11-09T06:16:00,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e24d31a2abfa45a591359951346601c4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e24d31a2abfa45a591359951346601c4 2024-11-09T06:16:00,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/73e4a19b9af14140adcbf4a699504f4c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/73e4a19b9af14140adcbf4a699504f4c 2024-11-09T06:16:00,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a69727a94b49ebb3deac2b7fa6ad09 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a69727a94b49ebb3deac2b7fa6ad09 2024-11-09T06:16:00,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/107b0887f79d42528f0771059783cf3b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/107b0887f79d42528f0771059783cf3b 2024-11-09T06:16:00,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c7dbc3d2fb4345f080b690e6dba261c6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c7dbc3d2fb4345f080b690e6dba261c6 2024-11-09T06:16:00,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3207768caa9045dfb8f733fd35b07920 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3207768caa9045dfb8f733fd35b07920 2024-11-09T06:16:00,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f08402b15fb7490d880fa1dd3c2c2346 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f08402b15fb7490d880fa1dd3c2c2346 2024-11-09T06:16:00,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2893ca04a2184478ad2bacb7ca491585 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2893ca04a2184478ad2bacb7ca491585 2024-11-09T06:16:00,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/23516da1264d42619fafec2d133182b0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/23516da1264d42619fafec2d133182b0 2024-11-09T06:16:00,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/138d416b0312472686924ae5d47250b8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/138d416b0312472686924ae5d47250b8 2024-11-09T06:16:00,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/5ec582759efd46f79081a52a13a09dba to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/5ec582759efd46f79081a52a13a09dba 2024-11-09T06:16:00,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2c085bf52d524902bd8d4c0c00ab4832 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/2c085bf52d524902bd8d4c0c00ab4832 2024-11-09T06:16:00,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f5a7a257aca14780bd80e490463ae81c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/f5a7a257aca14780bd80e490463ae81c 2024-11-09T06:16:00,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a6f849cd6c4ef39066654829a4b18c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e5a6f849cd6c4ef39066654829a4b18c 2024-11-09T06:16:00,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/dae1e09fb8354f8090f11666cad97445 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/dae1e09fb8354f8090f11666cad97445 2024-11-09T06:16:00,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/562424ed1cbb493eb75aa1970111a44d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/562424ed1cbb493eb75aa1970111a44d 2024-11-09T06:16:00,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3db0a2f21b594e2f89610bfdc0efc5bb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3db0a2f21b594e2f89610bfdc0efc5bb 2024-11-09T06:16:00,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/1502a82eb94b484780be9704350d5c63 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/1502a82eb94b484780be9704350d5c63 2024-11-09T06:16:00,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/b3847a17392242468033ad8593afb658 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/b3847a17392242468033ad8593afb658 2024-11-09T06:16:00,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/cfd4e3e477f949e09be9d64f7c4d6b21 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/cfd4e3e477f949e09be9d64f7c4d6b21 2024-11-09T06:16:00,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e9f8b762f2564962889476433c2f6e63 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/e9f8b762f2564962889476433c2f6e63 2024-11-09T06:16:00,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3883a2bb7c3b4751a7409acafb59eddf to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/3883a2bb7c3b4751a7409acafb59eddf 2024-11-09T06:16:00,551 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/recovered.edits/468.seqid, newMaxSeqId=468, maxSeqId=1 2024-11-09T06:16:00,552 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. 2024-11-09T06:16:00,553 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 4cec2ccd1a271d540c1f369d10f41e7f: Waiting for close lock at 1731132959032Running coprocessor pre-close hooks at 1731132959032Disabling compacts and flushes for region at 1731132959032Disabling writes for close at 1731132959032Obtaining lock to block concurrent updates at 1731132959033 (+1 ms)Preparing flush snapshotting stores in 4cec2ccd1a271d540c1f369d10f41e7f at 1731132959033Finished memstore snapshotting TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731132959034 (+1 ms)Flushing stores of TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f. at 1731132959034Flushing 4cec2ccd1a271d540c1f369d10f41e7f/A: creating writer at 1731132959034Flushing 4cec2ccd1a271d540c1f369d10f41e7f/A: appending metadata at 1731132959038 (+4 ms)Flushing 4cec2ccd1a271d540c1f369d10f41e7f/A: closing flushed file at 1731132959038Flushing 4cec2ccd1a271d540c1f369d10f41e7f/B: creating writer at 1731132959459 (+421 ms)Flushing 4cec2ccd1a271d540c1f369d10f41e7f/B: appending metadata at 1731132959463 (+4 ms)Flushing 4cec2ccd1a271d540c1f369d10f41e7f/B: closing flushed file at 1731132959463Flushing 4cec2ccd1a271d540c1f369d10f41e7f/C: creating writer at 1731132959874 (+411 ms)Flushing 4cec2ccd1a271d540c1f369d10f41e7f/C: appending metadata at 1731132959886 (+12 ms)Flushing 4cec2ccd1a271d540c1f369d10f41e7f/C: closing flushed file at 1731132959886Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5167cfa6: reopening flushed file at 1731132960299 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@419b3de3: reopening flushed file at 1731132960308 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c418b57: reopening flushed file at 1731132960318 (+10 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4cec2ccd1a271d540c1f369d10f41e7f in 1297ms, sequenceid=465, compaction requested=true at 1731132960329 (+11 ms)Writing region close event to WAL at 1731132960547 (+218 ms)Running coprocessor post-close hooks at 1731132960552 (+5 ms)Closed at 1731132960552 2024-11-09T06:16:00,556 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:16:00,556 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4cec2ccd1a271d540c1f369d10f41e7f, regionState=CLOSED 2024-11-09T06:16:00,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:16:00,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-09T06:16:00,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 4cec2ccd1a271d540c1f369d10f41e7f, server=4a84bcce7bad,46317,1731132929173 in 1.6870 sec 2024-11-09T06:16:00,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-09T06:16:00,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4cec2ccd1a271d540c1f369d10f41e7f, UNASSIGN in 1.6950 sec 2024-11-09T06:16:00,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-09T06:16:00,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7050 sec 2024-11-09T06:16:00,571 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731132960570"}]},"ts":"1731132960570"} 2024-11-09T06:16:00,572 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T06:16:00,573 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T06:16:00,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7210 sec 2024-11-09T06:16:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T06:16:01,003 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:01,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T06:16:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:01,013 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T06:16:01,016 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:01,021 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:16:01,025 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/recovered.edits] 2024-11-09T06:16:01,029 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b9798293f06647a09959977e3e2fc3e0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/b9798293f06647a09959977e3e2fc3e0 2024-11-09T06:16:01,031 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d8e65fad35fa4e42a990ca4689161280 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/d8e65fad35fa4e42a990ca4689161280 2024-11-09T06:16:01,034 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/faa6280f9a364feda2084481dfed9699 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/A/faa6280f9a364feda2084481dfed9699 2024-11-09T06:16:01,037 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/0be6a11468cf46418f0d0dcda17dd7bb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/0be6a11468cf46418f0d0dcda17dd7bb 2024-11-09T06:16:01,038 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/a5f61daf1590482e9bf82f2003ae502d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/a5f61daf1590482e9bf82f2003ae502d 2024-11-09T06:16:01,040 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d317b8a9ea0647e9b609d27319688cb7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/B/d317b8a9ea0647e9b609d27319688cb7 2024-11-09T06:16:01,043 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/6482e2fb08ec42d2a80ebfac7eb2db68 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/6482e2fb08ec42d2a80ebfac7eb2db68 2024-11-09T06:16:01,044 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c7c30f6791164748845520cd5ef3e545 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/c7c30f6791164748845520cd5ef3e545 2024-11-09T06:16:01,045 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/fbe6487af46d4c00bfb32d41686fdfd3 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/C/fbe6487af46d4c00bfb32d41686fdfd3 2024-11-09T06:16:01,048 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/recovered.edits/468.seqid to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f/recovered.edits/468.seqid 2024-11-09T06:16:01,049 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/4cec2ccd1a271d540c1f369d10f41e7f 2024-11-09T06:16:01,049 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T06:16:01,055 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:01,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-09T06:16:01,066 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T06:16:01,102 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T06:16:01,104 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:01,105 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T06:16:01,105 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731132961105"}]},"ts":"9223372036854775807"} 2024-11-09T06:16:01,109 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T06:16:01,109 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4cec2ccd1a271d540c1f369d10f41e7f, NAME => 'TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T06:16:01,109 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T06:16:01,110 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731132961109"}]},"ts":"9223372036854775807"} 2024-11-09T06:16:01,113 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T06:16:01,114 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:01,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 108 msec 2024-11-09T06:16:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T06:16:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T06:16:01,124 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:01,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-864101692_22 at /127.0.0.1:52142 [Waiting for operation #277] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043040278_22 at /127.0.0.1:56956 [Waiting for operation #103] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;4a84bcce7bad:46317-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-864101692_22 at /127.0.0.1:51750 [Waiting for operation #270] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=628 (was 343) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1201 (was 2429) 2024-11-09T06:16:01,151 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=628, ProcessCount=11, AvailableMemoryMB=1200 2024-11-09T06:16:01,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T06:16:01,154 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T06:16:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:01,157 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T06:16:01,157 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:01,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-11-09T06:16:01,159 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T06:16:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-09T06:16:01,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741942_1118 (size=1061) 2024-11-09T06:16:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-09T06:16:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-09T06:16:01,570 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f 2024-11-09T06:16:01,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741943_1119 (size=53) 2024-11-09T06:16:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-09T06:16:01,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:16:01,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f48863a85e472c28d23fb8419a1993ee, disabling compactions & flushes 2024-11-09T06:16:01,977 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:01,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:01,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. after waiting 0 ms 2024-11-09T06:16:01,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:01,977 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:01,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f48863a85e472c28d23fb8419a1993ee: Waiting for close lock at 1731132961977Disabling compacts and flushes for region at 1731132961977Disabling writes for close at 1731132961977Writing region close event to WAL at 1731132961977Closed at 1731132961977 2024-11-09T06:16:01,979 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T06:16:01,979 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731132961979"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731132961979"}]},"ts":"1731132961979"} 2024-11-09T06:16:01,982 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T06:16:01,983 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T06:16:01,983 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731132961983"}]},"ts":"1731132961983"} 2024-11-09T06:16:01,985 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T06:16:01,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, ASSIGN}] 2024-11-09T06:16:01,987 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, ASSIGN 2024-11-09T06:16:01,988 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, ASSIGN; state=OFFLINE, location=4a84bcce7bad,46317,1731132929173; forceNewPlan=false, retain=false 2024-11-09T06:16:02,139 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f48863a85e472c28d23fb8419a1993ee, regionState=OPENING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:02,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, ASSIGN because future has completed 2024-11-09T06:16:02,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:16:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-09T06:16:02,297 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:02,297 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} 2024-11-09T06:16:02,298 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,298 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:16:02,298 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,298 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,300 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,301 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:16:02,302 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f48863a85e472c28d23fb8419a1993ee columnFamilyName A 2024-11-09T06:16:02,302 DEBUG [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:02,302 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(327): Store=f48863a85e472c28d23fb8419a1993ee/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:16:02,302 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,304 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:16:02,304 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f48863a85e472c28d23fb8419a1993ee columnFamilyName B 2024-11-09T06:16:02,304 DEBUG [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:02,305 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(327): Store=f48863a85e472c28d23fb8419a1993ee/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:16:02,305 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,306 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:16:02,306 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f48863a85e472c28d23fb8419a1993ee columnFamilyName C 2024-11-09T06:16:02,306 DEBUG [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:02,307 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(327): Store=f48863a85e472c28d23fb8419a1993ee/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:16:02,307 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:02,307 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,308 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,308 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,310 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,310 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,310 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T06:16:02,312 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,314 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T06:16:02,315 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened f48863a85e472c28d23fb8419a1993ee; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65626089, jitterRate=-0.02209506928920746}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T06:16:02,315 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:02,316 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for f48863a85e472c28d23fb8419a1993ee: Running coprocessor pre-open hook at 1731132962298Writing region info on filesystem at 1731132962298Initializing all the Stores at 1731132962299 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132962299Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132962299Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132962299Cleaning up temporary data from old regions at 1731132962310 (+11 ms)Running coprocessor post-open hooks at 1731132962315 (+5 ms)Region opened successfully at 1731132962316 (+1 ms) 2024-11-09T06:16:02,318 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., pid=28, masterSystemTime=1731132962294 2024-11-09T06:16:02,320 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:02,320 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:02,321 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f48863a85e472c28d23fb8419a1993ee, regionState=OPEN, openSeqNum=2, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:02,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:16:02,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-09T06:16:02,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 in 184 msec 2024-11-09T06:16:02,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-09T06:16:02,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, ASSIGN in 342 msec 2024-11-09T06:16:02,331 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T06:16:02,331 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731132962331"}]},"ts":"1731132962331"} 2024-11-09T06:16:02,333 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T06:16:02,334 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T06:16:02,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-09T06:16:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-09T06:16:03,304 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:03,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a00013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:03,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:03,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:03,307 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:03,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:03,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:03,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1896e41f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:03,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:03,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:03,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:03,310 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641dcc92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:03,311 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:03,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:03,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:03,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:03,316 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T06:16:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a84bcce7bad,37069,1731132928314 2024-11-09T06:16:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@296fe578 2024-11-09T06:16:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T06:16:03,318 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T06:16:03,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T06:16:03,324 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T06:16:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:03,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741944_1120 (size=1097) 2024-11-09T06:16:03,747 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-09T06:16:03,747 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-09T06:16:03,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T06:16:03,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, REOPEN/MOVE}] 2024-11-09T06:16:03,763 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, REOPEN/MOVE 2024-11-09T06:16:03,764 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f48863a85e472c28d23fb8419a1993ee, regionState=CLOSING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:03,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, REOPEN/MOVE because future has completed 2024-11-09T06:16:03,766 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T06:16:03,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:16:03,919 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:03,919 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T06:16:03,919 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing f48863a85e472c28d23fb8419a1993ee, disabling compactions & flushes 2024-11-09T06:16:03,919 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:03,919 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:03,920 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. after waiting 0 ms 2024-11-09T06:16:03,920 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:03,924 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T06:16:03,924 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:03,924 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for f48863a85e472c28d23fb8419a1993ee: Waiting for close lock at 1731132963919Running coprocessor pre-close hooks at 1731132963919Disabling compacts and flushes for region at 1731132963919Disabling writes for close at 1731132963920 (+1 ms)Writing region close event to WAL at 1731132963920Running coprocessor post-close hooks at 1731132963924 (+4 ms)Closed at 1731132963924 2024-11-09T06:16:03,925 WARN [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: f48863a85e472c28d23fb8419a1993ee to self. 2024-11-09T06:16:03,927 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:03,927 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f48863a85e472c28d23fb8419a1993ee, regionState=CLOSED 2024-11-09T06:16:03,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:16:03,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-09T06:16:03,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 in 165 msec 2024-11-09T06:16:03,934 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, REOPEN/MOVE; state=CLOSED, location=4a84bcce7bad,46317,1731132929173; forceNewPlan=false, retain=true 2024-11-09T06:16:04,085 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f48863a85e472c28d23fb8419a1993ee, regionState=OPENING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, REOPEN/MOVE because future has completed 2024-11-09T06:16:04,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:16:04,246 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,246 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} 2024-11-09T06:16:04,246 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,247 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:16:04,247 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,247 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,251 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,252 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:16:04,258 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f48863a85e472c28d23fb8419a1993ee columnFamilyName A 2024-11-09T06:16:04,261 DEBUG [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:04,261 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(327): Store=f48863a85e472c28d23fb8419a1993ee/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:16:04,262 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,262 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:16:04,263 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f48863a85e472c28d23fb8419a1993ee columnFamilyName B 2024-11-09T06:16:04,263 DEBUG [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:04,263 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(327): Store=f48863a85e472c28d23fb8419a1993ee/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:16:04,263 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,264 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:16:04,264 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f48863a85e472c28d23fb8419a1993ee columnFamilyName C 2024-11-09T06:16:04,264 DEBUG [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:04,265 INFO [StoreOpener-f48863a85e472c28d23fb8419a1993ee-1 {}] regionserver.HStore(327): Store=f48863a85e472c28d23fb8419a1993ee/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:16:04,265 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,265 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,266 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,267 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,268 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,268 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,269 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T06:16:04,270 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,271 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened f48863a85e472c28d23fb8419a1993ee; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74160974, jitterRate=0.105084627866745}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T06:16:04,271 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,273 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for f48863a85e472c28d23fb8419a1993ee: Running coprocessor pre-open hook at 1731132964247Writing region info on filesystem at 1731132964247Initializing all the Stores at 1731132964248 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132964248Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132964248Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132964248Cleaning up temporary data from old regions at 1731132964268 (+20 ms)Running coprocessor post-open hooks at 1731132964271 (+3 ms)Region opened successfully at 1731132964273 (+2 ms) 2024-11-09T06:16:04,274 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., pid=33, masterSystemTime=1731132964240 2024-11-09T06:16:04,276 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,276 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,277 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f48863a85e472c28d23fb8419a1993ee, regionState=OPEN, openSeqNum=5, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:16:04,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-11-09T06:16:04,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 in 192 msec 2024-11-09T06:16:04,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-09T06:16:04,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, REOPEN/MOVE in 520 msec 2024-11-09T06:16:04,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-09T06:16:04,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 534 msec 2024-11-09T06:16:04,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 963 msec 2024-11-09T06:16:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T06:16:04,295 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:04,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad8b864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:04,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:04,297 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:04,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:04,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:04,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6312b5bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:04,300 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9f6e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:04,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:04,304 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2c867e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:04,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:04,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e45b440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:04,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:04,311 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:04,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:04,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:04,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f4723c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:04,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:04,313 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:04,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a9ed6c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,315 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:04,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:04,316 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65843f7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:04,318 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e54ba6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:04,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:04,321 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:04,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7add911c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:04,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:04,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:04,323 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:04,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c4b81d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:04,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:04,327 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ad90c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:04,329 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:04,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540da924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:04,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:04,332 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d875f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:04,334 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:04,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0905f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:04,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:04,337 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c3e521, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:04,340 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f1458f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:04,342 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:04,343 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c58ba93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:04,345 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442cd356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:04,347 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:04,347 DEBUG [Thread-619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:04,347 DEBUG [Thread-617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:04,347 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:04,348 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:04,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:04,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:04,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-09T06:16:04,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:04,353 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T06:16:04,354 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:04,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:16:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:16:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:16:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:16:04,363 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:04,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:04,366 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:04,366 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:04,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:04,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:04,368 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:04,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:04,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:04,369 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:04,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:16:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:04,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:16:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:16:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:16:04,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:04,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:04,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:04,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:04,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:04,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:04,377 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:16:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:16:04,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090fc52b61fd6d48e4a24967115d5e8dbd_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132964372/Put/seqid=0 2024-11-09T06:16:04,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133024433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133024437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731132964397, max timestamp=1731132964439Num uniques 859; 2024-11-09T06:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T06:16:04,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731132964397, max timestamp=1731132964439Num uniques 859; 2024-11-09T06:16:04,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T06:16:04,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741945_1121 (size=17034) 2024-11-09T06:16:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T06:16:04,463 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:04,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731132964420, max timestamp=1731132964439Num uniques 859; 2024-11-09T06:16:04,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:04,489 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090fc52b61fd6d48e4a24967115d5e8dbd_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090fc52b61fd6d48e4a24967115d5e8dbd_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:04,491 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/ea2879829ce94a18b39755f24e997407, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:04,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/ea2879829ce94a18b39755f24e997407 is 175, key is test_row_0/A:col10/1731132964372/Put/seqid=0 2024-11-09T06:16:04,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-09T06:16:04,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:04,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133024528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133024539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133024540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741946_1122 (size=48139) 2024-11-09T06:16:04,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133024563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133024564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133024643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133024656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133024658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-09T06:16:04,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:04,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T06:16:04,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133024774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133024774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-09T06:16:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133024862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133024862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:04,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133024863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:04,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:04,955 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/ea2879829ce94a18b39755f24e997407 2024-11-09T06:16:04,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-09T06:16:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:04,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:04,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T06:16:05,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/ef771a86c19544fd8a5194a4d0a7a0ff is 50, key is test_row_0/B:col10/1731132964372/Put/seqid=0 2024-11-09T06:16:05,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741947_1123 (size=12001) 2024-11-09T06:16:05,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/ef771a86c19544fd8a5194a4d0a7a0ff 2024-11-09T06:16:05,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/8569f62916f948b5b634bcb555a6c3a3 is 50, key is test_row_0/C:col10/1731132964372/Put/seqid=0 2024-11-09T06:16:05,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133025082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133025082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-09T06:16:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:05,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:05,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741948_1124 (size=12001) 2024-11-09T06:16:05,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/8569f62916f948b5b634bcb555a6c3a3 2024-11-09T06:16:05,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/ea2879829ce94a18b39755f24e997407 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ea2879829ce94a18b39755f24e997407 2024-11-09T06:16:05,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ea2879829ce94a18b39755f24e997407, entries=250, sequenceid=14, filesize=47.0 K 2024-11-09T06:16:05,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/ef771a86c19544fd8a5194a4d0a7a0ff as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/ef771a86c19544fd8a5194a4d0a7a0ff 2024-11-09T06:16:05,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/ef771a86c19544fd8a5194a4d0a7a0ff, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T06:16:05,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133025182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133025183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133025183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/8569f62916f948b5b634bcb555a6c3a3 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8569f62916f948b5b634bcb555a6c3a3 2024-11-09T06:16:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8569f62916f948b5b634bcb555a6c3a3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T06:16:05,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for f48863a85e472c28d23fb8419a1993ee in 853ms, sequenceid=14, compaction requested=false 2024-11-09T06:16:05,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:05,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-09T06:16:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:05,288 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=138.65 KB heapSize=385.77 KB 2024-11-09T06:16:05,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:05,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:05,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:05,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:05,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:05,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:05,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a5d603e404424234881ff61399b5eec4_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132964526/Put/seqid=0 2024-11-09T06:16:05,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741949_1125 (size=12154) 2024-11-09T06:16:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T06:16:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:05,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:05,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133025637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133025639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133025723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133025723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133025724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133025745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:05,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133025753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,809 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a5d603e404424234881ff61399b5eec4_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a5d603e404424234881ff61399b5eec4_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:05,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/2fbea6b644ae46b2a6f8a20bab634857, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:05,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/2fbea6b644ae46b2a6f8a20bab634857 is 175, key is test_row_0/A:col10/1731132964526/Put/seqid=0 2024-11-09T06:16:05,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741950_1126 (size=30955) 2024-11-09T06:16:05,869 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/2fbea6b644ae46b2a6f8a20bab634857 2024-11-09T06:16:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/6cdac06a2c344419a71fde6286ff7178 is 50, key is test_row_0/B:col10/1731132964526/Put/seqid=0 2024-11-09T06:16:05,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741951_1127 (size=12001) 2024-11-09T06:16:05,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133025962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:05,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133025995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,133 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T06:16:06,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133026284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133026304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,359 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/6cdac06a2c344419a71fde6286ff7178 2024-11-09T06:16:06,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/a26f12e3354543d6a27d246a95ba65ab is 50, key is test_row_0/C:col10/1731132964526/Put/seqid=0 2024-11-09T06:16:06,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741952_1128 (size=12001) 2024-11-09T06:16:06,450 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/a26f12e3354543d6a27d246a95ba65ab 2024-11-09T06:16:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/2fbea6b644ae46b2a6f8a20bab634857 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fbea6b644ae46b2a6f8a20bab634857 2024-11-09T06:16:06,482 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fbea6b644ae46b2a6f8a20bab634857, entries=150, sequenceid=49, filesize=30.2 K 2024-11-09T06:16:06,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/6cdac06a2c344419a71fde6286ff7178 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6cdac06a2c344419a71fde6286ff7178 2024-11-09T06:16:06,502 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6cdac06a2c344419a71fde6286ff7178, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T06:16:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T06:16:06,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/a26f12e3354543d6a27d246a95ba65ab as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/a26f12e3354543d6a27d246a95ba65ab 2024-11-09T06:16:06,521 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/a26f12e3354543d6a27d246a95ba65ab, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T06:16:06,527 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~385.72 KB/394976, currentSize=46.96 KB/48090 for f48863a85e472c28d23fb8419a1993ee in 1235ms, sequenceid=49, compaction requested=false 2024-11-09T06:16:06,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:06,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:06,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-09T06:16:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-09T06:16:06,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-09T06:16:06,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1780 sec 2024-11-09T06:16:06,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 2.1920 sec 2024-11-09T06:16:06,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T06:16:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:06,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:06,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:06,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:06,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:06,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:06,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:06,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109eeb072ab3e6647a9903454886a995544_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132965635/Put/seqid=0 2024-11-09T06:16:06,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741953_1129 (size=14594) 2024-11-09T06:16:06,835 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:06,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133026825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,847 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109eeb072ab3e6647a9903454886a995544_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eeb072ab3e6647a9903454886a995544_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:06,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133026829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,851 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/0f57966d4ff74f46b37c861c5e8c5096, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:06,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/0f57966d4ff74f46b37c861c5e8c5096 is 175, key is test_row_0/A:col10/1731132965635/Put/seqid=0 2024-11-09T06:16:06,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133026842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133026844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133026844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741954_1130 (size=39549) 2024-11-09T06:16:06,910 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/0f57966d4ff74f46b37c861c5e8c5096 2024-11-09T06:16:06,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/2f9005a2f62e49d7ac8e885791c46139 is 50, key is test_row_0/B:col10/1731132965635/Put/seqid=0 2024-11-09T06:16:06,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133026953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133026953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741955_1131 (size=12001) 2024-11-09T06:16:06,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133026963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133026964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:06,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133026965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:07,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133027163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133027163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:07,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133027183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:07,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133027193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:07,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133027193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:07,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:07,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/2f9005a2f62e49d7ac8e885791c46139 2024-11-09T06:16:07,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/94b2fdfc84b548eda306eb99fd611a30 is 50, key is test_row_0/C:col10/1731132965635/Put/seqid=0 2024-11-09T06:16:07,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741956_1132 (size=12001) 2024-11-09T06:16:07,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/94b2fdfc84b548eda306eb99fd611a30 2024-11-09T06:16:07,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/0f57966d4ff74f46b37c861c5e8c5096 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/0f57966d4ff74f46b37c861c5e8c5096 2024-11-09T06:16:07,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/0f57966d4ff74f46b37c861c5e8c5096, entries=200, sequenceid=61, filesize=38.6 K 2024-11-09T06:16:07,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/2f9005a2f62e49d7ac8e885791c46139 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/2f9005a2f62e49d7ac8e885791c46139 2024-11-09T06:16:07,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/2f9005a2f62e49d7ac8e885791c46139, entries=150, sequenceid=61, filesize=11.7 K 2024-11-09T06:16:07,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/94b2fdfc84b548eda306eb99fd611a30 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/94b2fdfc84b548eda306eb99fd611a30 2024-11-09T06:16:07,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/94b2fdfc84b548eda306eb99fd611a30, entries=150, sequenceid=61, filesize=11.7 K 2024-11-09T06:16:07,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f48863a85e472c28d23fb8419a1993ee in 736ms, sequenceid=61, compaction requested=true 2024-11-09T06:16:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:07,485 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:07,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T06:16:07,486 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:07,487 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:07,487 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:07,488 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/A is initiating minor compaction (all files) 2024-11-09T06:16:07,488 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/B is initiating minor compaction (all files) 2024-11-09T06:16:07,488 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/B in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:07,488 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/A in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:07,488 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/ef771a86c19544fd8a5194a4d0a7a0ff, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6cdac06a2c344419a71fde6286ff7178, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/2f9005a2f62e49d7ac8e885791c46139] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=35.2 K 2024-11-09T06:16:07,488 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ea2879829ce94a18b39755f24e997407, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fbea6b644ae46b2a6f8a20bab634857, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/0f57966d4ff74f46b37c861c5e8c5096] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=115.9 K 2024-11-09T06:16:07,488 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:07,488 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting ef771a86c19544fd8a5194a4d0a7a0ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731132964362 2024-11-09T06:16:07,488 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ea2879829ce94a18b39755f24e997407, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fbea6b644ae46b2a6f8a20bab634857, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/0f57966d4ff74f46b37c861c5e8c5096] 2024-11-09T06:16:07,489 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea2879829ce94a18b39755f24e997407, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731132964362 2024-11-09T06:16:07,489 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cdac06a2c344419a71fde6286ff7178, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731132964516 2024-11-09T06:16:07,489 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fbea6b644ae46b2a6f8a20bab634857, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731132964516 2024-11-09T06:16:07,490 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9005a2f62e49d7ac8e885791c46139, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731132965635 2024-11-09T06:16:07,490 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f57966d4ff74f46b37c861c5e8c5096, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731132965624 2024-11-09T06:16:07,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-09T06:16:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:07,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132966822, max timestamp=1731132967491Num uniques 859; 2024-11-09T06:16:07,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:07,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132966822, max timestamp=1731132967491Num uniques 859; 2024-11-09T06:16:07,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:07,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132966822, max timestamp=1731132967491Num uniques 859; 2024-11-09T06:16:07,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:07,541 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:07,568 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:07,569 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/5ec3948c3300409eb09dec9f611ae093 is 50, key is test_row_0/B:col10/1731132965635/Put/seqid=0 2024-11-09T06:16:07,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110961f5a1b26db147e490f4189c1b17c2fa_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132967493/Put/seqid=0 2024-11-09T06:16:07,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731132967557, max timestamp=1731132967584Num uniques 859; 2024-11-09T06:16:07,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-09T06:16:07,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731132967557, max timestamp=1731132967584Num uniques 859; 2024-11-09T06:16:07,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-09T06:16:07,593 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411098a6e657ea6724cca93e3cd489e0468e9_f48863a85e472c28d23fb8419a1993ee store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:07,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731132967557, max timestamp=1731132967584Num uniques 859; 2024-11-09T06:16:07,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-09T06:16:07,602 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411098a6e657ea6724cca93e3cd489e0468e9_f48863a85e472c28d23fb8419a1993ee, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:07,603 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098a6e657ea6724cca93e3cd489e0468e9_f48863a85e472c28d23fb8419a1993ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:07,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741958_1134 (size=12154) 2024-11-09T06:16:07,662 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:07,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741957_1133 (size=12104) 2024-11-09T06:16:07,673 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110961f5a1b26db147e490f4189c1b17c2fa_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110961f5a1b26db147e490f4189c1b17c2fa_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:07,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=418, cellSize=19138, totalHeapSize=43712, min timestamp=1731132967635, max timestamp=1731132967671Num uniques 1009; 2024-11-09T06:16:07,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=19138, suffix heap size=132904,new segment heap size=43712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=418 2024-11-09T06:16:07,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=418, cellSize=19138, totalHeapSize=43712, min timestamp=1731132967635, max timestamp=1731132967671Num uniques 1009; 2024-11-09T06:16:07,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=19138, suffix heap size=132904,new segment heap size=43712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=418 2024-11-09T06:16:07,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731132967637, max timestamp=1731132967671Num uniques 1009; 2024-11-09T06:16:07,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-11-09T06:16:07,682 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/48c1f54e0adc42d9aaf09af240989d2c, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:07,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741959_1135 (size=4469) 2024-11-09T06:16:07,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/48c1f54e0adc42d9aaf09af240989d2c is 175, key is test_row_0/A:col10/1731132967493/Put/seqid=0 2024-11-09T06:16:07,684 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#A#compaction#114 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:07,685 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/5d61eca7284c42959015cc4d54da0563 is 175, key is test_row_0/A:col10/1731132965635/Put/seqid=0 2024-11-09T06:16:07,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741960_1136 (size=30955) 2024-11-09T06:16:07,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/48c1f54e0adc42d9aaf09af240989d2c 2024-11-09T06:16:07,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741961_1137 (size=31058) 2024-11-09T06:16:07,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/6463b345fc034a76b4312c8d2b6e8842 is 50, key is test_row_0/B:col10/1731132967493/Put/seqid=0 2024-11-09T06:16:07,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731132967706, max timestamp=1731132967750Num uniques 1277; 2024-11-09T06:16:07,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=58482, new segment data size=14972, suffix heap size=160776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1277, new segment cells count=327 2024-11-09T06:16:07,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731132967706, max timestamp=1731132967750Num uniques 1277; 2024-11-09T06:16:07,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=58482, new segment data size=14972, suffix heap size=160776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1277, new segment cells count=327 2024-11-09T06:16:07,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741962_1138 (size=12001) 2024-11-09T06:16:07,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/6463b345fc034a76b4312c8d2b6e8842 2024-11-09T06:16:07,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731132967722, max timestamp=1731132967765Num uniques 1127; 2024-11-09T06:16:07,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-09T06:16:07,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/dc349f63c2c349f281e37d833cb1c770 is 50, key is test_row_0/C:col10/1731132967493/Put/seqid=0 2024-11-09T06:16:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741963_1139 (size=12001) 2024-11-09T06:16:07,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/dc349f63c2c349f281e37d833cb1c770 2024-11-09T06:16:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/48c1f54e0adc42d9aaf09af240989d2c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48c1f54e0adc42d9aaf09af240989d2c 2024-11-09T06:16:07,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48c1f54e0adc42d9aaf09af240989d2c, entries=150, sequenceid=83, filesize=30.2 K 2024-11-09T06:16:07,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/6463b345fc034a76b4312c8d2b6e8842 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6463b345fc034a76b4312c8d2b6e8842 2024-11-09T06:16:07,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6463b345fc034a76b4312c8d2b6e8842, entries=150, sequenceid=83, filesize=11.7 K 2024-11-09T06:16:07,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/dc349f63c2c349f281e37d833cb1c770 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dc349f63c2c349f281e37d833cb1c770 2024-11-09T06:16:07,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dc349f63c2c349f281e37d833cb1c770, entries=150, sequenceid=83, filesize=11.7 K 2024-11-09T06:16:07,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=136.42 KB/139690 for f48863a85e472c28d23fb8419a1993ee in 410ms, sequenceid=83, compaction requested=true 2024-11-09T06:16:07,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:A, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T06:16:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:B, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-09T06:16:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-09T06:16:07,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=143.13 KB heapSize=398.92 KB 2024-11-09T06:16:07,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:07,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:07,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:07,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:07,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:07,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:07,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b8e300ea6a904baa8cf15a8c05b765f4_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132967845/Put/seqid=0 2024-11-09T06:16:07,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133027942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:07,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133027945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:07,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133027951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741964_1140 (size=12304) 2024-11-09T06:16:08,002 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:08,009 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b8e300ea6a904baa8cf15a8c05b765f4_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b8e300ea6a904baa8cf15a8c05b765f4_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:08,011 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/978f41cd4f8f47e8bc28beb2536cf16b, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:08,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/978f41cd4f8f47e8bc28beb2536cf16b is 175, key is test_row_0/A:col10/1731132967845/Put/seqid=0 2024-11-09T06:16:08,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133027981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133027983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741965_1141 (size=31105) 2024-11-09T06:16:08,075 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/5ec3948c3300409eb09dec9f611ae093 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/5ec3948c3300409eb09dec9f611ae093 2024-11-09T06:16:08,084 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/B of f48863a85e472c28d23fb8419a1993ee into 5ec3948c3300409eb09dec9f611ae093(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:08,084 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:08,084 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/B, priority=13, startTime=1731132967485; duration=0sec 2024-11-09T06:16:08,084 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-09T06:16:08,084 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:B 2024-11-09T06:16:08,084 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-09T06:16:08,085 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T06:16:08,085 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T06:16:08,085 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. because compaction request was cancelled 2024-11-09T06:16:08,085 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:A 2024-11-09T06:16:08,085 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:16:08,087 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:16:08,087 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/C is initiating minor compaction (all files) 2024-11-09T06:16:08,087 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/C in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:08,088 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8569f62916f948b5b634bcb555a6c3a3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/a26f12e3354543d6a27d246a95ba65ab, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/94b2fdfc84b548eda306eb99fd611a30, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dc349f63c2c349f281e37d833cb1c770] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=46.9 K 2024-11-09T06:16:08,088 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8569f62916f948b5b634bcb555a6c3a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731132964362 2024-11-09T06:16:08,090 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a26f12e3354543d6a27d246a95ba65ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731132964516 2024-11-09T06:16:08,090 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b2fdfc84b548eda306eb99fd611a30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731132965635 2024-11-09T06:16:08,091 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting dc349f63c2c349f281e37d833cb1c770, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731132966827 2024-11-09T06:16:08,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133028094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133028094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133028103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133028133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133028134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,150 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#C#compaction#120 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:08,152 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/1b357a37af844b68af29b85261bcb5f9 is 50, key is test_row_0/C:col10/1731132967493/Put/seqid=0 2024-11-09T06:16:08,159 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/5d61eca7284c42959015cc4d54da0563 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5d61eca7284c42959015cc4d54da0563 2024-11-09T06:16:08,172 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/A of f48863a85e472c28d23fb8419a1993ee into 5d61eca7284c42959015cc4d54da0563(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:08,172 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:08,172 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/A, priority=13, startTime=1731132967485; duration=0sec 2024-11-09T06:16:08,172 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T06:16:08,172 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:A 2024-11-09T06:16:08,172 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:B 2024-11-09T06:16:08,172 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-09T06:16:08,172 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T06:16:08,172 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T06:16:08,172 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. because compaction request was cancelled 2024-11-09T06:16:08,172 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:C 2024-11-09T06:16:08,172 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T06:16:08,175 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T06:16:08,175 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T06:16:08,175 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. because compaction request was cancelled 2024-11-09T06:16:08,175 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:B 2024-11-09T06:16:08,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741966_1142 (size=12139) 2024-11-09T06:16:08,206 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/1b357a37af844b68af29b85261bcb5f9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/1b357a37af844b68af29b85261bcb5f9 2024-11-09T06:16:08,217 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/C of f48863a85e472c28d23fb8419a1993ee into 1b357a37af844b68af29b85261bcb5f9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:08,217 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:08,217 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/C, priority=12, startTime=1731132967913; duration=0sec 2024-11-09T06:16:08,218 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:08,218 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:C 2024-11-09T06:16:08,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133028303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133028313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133028333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133028343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133028354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,473 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/978f41cd4f8f47e8bc28beb2536cf16b 2024-11-09T06:16:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T06:16:08,524 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:08,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/28ac651612284316b2bcd149745884bf is 50, key is test_row_0/B:col10/1731132967845/Put/seqid=0 2024-11-09T06:16:08,527 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-09T06:16:08,530 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:08,531 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:08,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-09T06:16:08,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741967_1143 (size=12151) 2024-11-09T06:16:08,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/28ac651612284316b2bcd149745884bf 2024-11-09T06:16:08,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/dcc6ae97728a4743be5090f5ab9925de is 50, key is test_row_0/C:col10/1731132967845/Put/seqid=0 2024-11-09T06:16:08,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741968_1144 (size=12151) 2024-11-09T06:16:08,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/dcc6ae97728a4743be5090f5ab9925de 2024-11-09T06:16:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133028625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133028627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-09T06:16:08,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/978f41cd4f8f47e8bc28beb2536cf16b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/978f41cd4f8f47e8bc28beb2536cf16b 2024-11-09T06:16:08,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/978f41cd4f8f47e8bc28beb2536cf16b, entries=150, sequenceid=154, filesize=30.4 K 2024-11-09T06:16:08,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/28ac651612284316b2bcd149745884bf as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/28ac651612284316b2bcd149745884bf 2024-11-09T06:16:08,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133028653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133028664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133028673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:08,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/28ac651612284316b2bcd149745884bf, entries=150, sequenceid=154, filesize=11.9 K 2024-11-09T06:16:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:08,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/dcc6ae97728a4743be5090f5ab9925de as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dcc6ae97728a4743be5090f5ab9925de 2024-11-09T06:16:08,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-09T06:16:08,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:08,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:08,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:08,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:08,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:08,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dcc6ae97728a4743be5090f5ab9925de, entries=150, sequenceid=154, filesize=11.9 K 2024-11-09T06:16:08,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~149.83 KB/153430, heapSize ~418.80 KB/428848, currentSize=40.25 KB/41220 for f48863a85e472c28d23fb8419a1993ee in 798ms, sequenceid=154, compaction requested=true 2024-11-09T06:16:08,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:08,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:08,713 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:08,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:08,714 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:08,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:08,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:08,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:08,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:08,715 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93118 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:08,715 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/A is initiating minor compaction (all files) 2024-11-09T06:16:08,716 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/A in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:08,716 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5d61eca7284c42959015cc4d54da0563, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48c1f54e0adc42d9aaf09af240989d2c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/978f41cd4f8f47e8bc28beb2536cf16b] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=90.9 K 2024-11-09T06:16:08,716 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:08,716 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5d61eca7284c42959015cc4d54da0563, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48c1f54e0adc42d9aaf09af240989d2c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/978f41cd4f8f47e8bc28beb2536cf16b] 2024-11-09T06:16:08,717 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d61eca7284c42959015cc4d54da0563, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731132965635 2024-11-09T06:16:08,717 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:08,717 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/B is initiating minor compaction (all files) 2024-11-09T06:16:08,717 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/B in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:08,718 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c1f54e0adc42d9aaf09af240989d2c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731132966827 2024-11-09T06:16:08,718 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/5ec3948c3300409eb09dec9f611ae093, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6463b345fc034a76b4312c8d2b6e8842, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/28ac651612284316b2bcd149745884bf] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=35.4 K 2024-11-09T06:16:08,718 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 978f41cd4f8f47e8bc28beb2536cf16b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731132967845 2024-11-09T06:16:08,718 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ec3948c3300409eb09dec9f611ae093, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731132965635 2024-11-09T06:16:08,719 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6463b345fc034a76b4312c8d2b6e8842, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731132966827 2024-11-09T06:16:08,721 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ac651612284316b2bcd149745884bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731132967845 2024-11-09T06:16:08,743 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:08,758 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/af8cdc71e7d5423eb88b0ff2023cd26c is 50, key is test_row_0/B:col10/1731132967845/Put/seqid=0 2024-11-09T06:16:08,761 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:08,781 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411096c5724150f0f4b45a85d7183f01c1cd9_f48863a85e472c28d23fb8419a1993ee store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:08,798 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411096c5724150f0f4b45a85d7183f01c1cd9_f48863a85e472c28d23fb8419a1993ee, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:08,798 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096c5724150f0f4b45a85d7183f01c1cd9_f48863a85e472c28d23fb8419a1993ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:08,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741969_1145 (size=12357) 2024-11-09T06:16:08,826 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/af8cdc71e7d5423eb88b0ff2023cd26c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/af8cdc71e7d5423eb88b0ff2023cd26c 2024-11-09T06:16:08,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-09T06:16:08,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:08,848 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-09T06:16:08,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:08,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:08,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:08,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:08,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:08,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:08,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741970_1146 (size=4469) 2024-11-09T06:16:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-09T06:16:08,855 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#A#compaction#124 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:08,855 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/b179bc44e54443f49b22861632d3bff9 is 175, key is test_row_0/A:col10/1731132967845/Put/seqid=0 2024-11-09T06:16:08,860 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/B of f48863a85e472c28d23fb8419a1993ee into af8cdc71e7d5423eb88b0ff2023cd26c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:08,860 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:08,860 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/B, priority=13, startTime=1731132968713; duration=0sec 2024-11-09T06:16:08,860 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:08,860 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:B 2024-11-09T06:16:08,860 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T06:16:08,863 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T06:16:08,863 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T06:16:08,864 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. because compaction request was cancelled 2024-11-09T06:16:08,864 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:C 2024-11-09T06:16:08,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097b8f0ec35a114b49b01651530170c789_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132967973/Put/seqid=0 2024-11-09T06:16:08,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741971_1147 (size=31311) 2024-11-09T06:16:08,946 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/b179bc44e54443f49b22861632d3bff9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/b179bc44e54443f49b22861632d3bff9 2024-11-09T06:16:08,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741972_1148 (size=12304) 2024-11-09T06:16:08,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:08,971 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/A of f48863a85e472c28d23fb8419a1993ee into b179bc44e54443f49b22861632d3bff9(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:08,971 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:08,971 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/A, priority=13, startTime=1731132968713; duration=0sec 2024-11-09T06:16:08,971 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:08,971 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:A 2024-11-09T06:16:09,000 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097b8f0ec35a114b49b01651530170c789_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097b8f0ec35a114b49b01651530170c789_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/48e7f92988484ee6bb6a5d0955742db9, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:09,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/48e7f92988484ee6bb6a5d0955742db9 is 175, key is test_row_0/A:col10/1731132967973/Put/seqid=0 2024-11-09T06:16:09,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741973_1149 (size=31105) 2024-11-09T06:16:09,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-09T06:16:09,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731132969165, max timestamp=1731132969226Num uniques 859; 2024-11-09T06:16:09,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731132969165, max timestamp=1731132969226Num uniques 859; 2024-11-09T06:16:09,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-09T06:16:09,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-09T06:16:09,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731132969165, max timestamp=1731132969226Num uniques 859; 2024-11-09T06:16:09,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-09T06:16:09,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133029278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133029279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133029288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133029292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133029318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133029403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133029404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133029423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133029423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133029433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,461 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/48e7f92988484ee6bb6a5d0955742db9 2024-11-09T06:16:09,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/a108e19c5d33404091c2b75bb234138e is 50, key is test_row_0/B:col10/1731132967973/Put/seqid=0 2024-11-09T06:16:09,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741974_1150 (size=12151) 2024-11-09T06:16:09,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133029623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133029623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133029633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133029634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133029652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-09T06:16:09,936 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/a108e19c5d33404091c2b75bb234138e 2024-11-09T06:16:09,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133029933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133029934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133029947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133029950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133029968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:09,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/7153a6a1539746009e8250018772f713 is 50, key is test_row_0/C:col10/1731132967973/Put/seqid=0 2024-11-09T06:16:10,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741975_1151 (size=12151) 2024-11-09T06:16:10,035 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/7153a6a1539746009e8250018772f713 2024-11-09T06:16:10,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/48e7f92988484ee6bb6a5d0955742db9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48e7f92988484ee6bb6a5d0955742db9 2024-11-09T06:16:10,074 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48e7f92988484ee6bb6a5d0955742db9, entries=150, sequenceid=167, filesize=30.4 K 2024-11-09T06:16:10,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/a108e19c5d33404091c2b75bb234138e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a108e19c5d33404091c2b75bb234138e 2024-11-09T06:16:10,087 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a108e19c5d33404091c2b75bb234138e, entries=150, sequenceid=167, filesize=11.9 K 2024-11-09T06:16:10,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/7153a6a1539746009e8250018772f713 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/7153a6a1539746009e8250018772f713 2024-11-09T06:16:10,105 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/7153a6a1539746009e8250018772f713, entries=150, sequenceid=167, filesize=11.9 K 2024-11-09T06:16:10,111 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=154.31 KB/158010 for f48863a85e472c28d23fb8419a1993ee in 1263ms, sequenceid=167, compaction requested=true 2024-11-09T06:16:10,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:10,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:10,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-09T06:16:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-09T06:16:10,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-09T06:16:10,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5870 sec 2024-11-09T06:16:10,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.6050 sec 2024-11-09T06:16:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:10,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=161.02 KB heapSize=431.86 KB 2024-11-09T06:16:10,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:10,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:10,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:10,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:10,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:10,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:10,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093b8364c3fa494781a909c761a200983a_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132969290/Put/seqid=0 2024-11-09T06:16:10,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133030490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133030490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133030491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133030495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133030498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741976_1152 (size=14794) 2024-11-09T06:16:10,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133030604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133030604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133030605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133030605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-09T06:16:10,694 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:10,694 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-09T06:16:10,697 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T06:16:10,702 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:10,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T06:16:10,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133030813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133030814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133030823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133030823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:10,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:10,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T06:16:10,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:10,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:10,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:10,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:10,961 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:10,967 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093b8364c3fa494781a909c761a200983a_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093b8364c3fa494781a909c761a200983a_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:10,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/f8dc3aeffb9545a3bf122452ef838532, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:10,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/f8dc3aeffb9545a3bf122452ef838532 is 175, key is test_row_0/A:col10/1731132969290/Put/seqid=0 2024-11-09T06:16:10,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741977_1153 (size=39749) 2024-11-09T06:16:10,989 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/f8dc3aeffb9545a3bf122452ef838532 2024-11-09T06:16:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T06:16:11,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T06:16:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/b09d4e92b44c4c3482ee3dc5209c8f15 is 50, key is test_row_0/B:col10/1731132969290/Put/seqid=0 2024-11-09T06:16:11,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741978_1154 (size=12151) 2024-11-09T06:16:11,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133031123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:11,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133031123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:11,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:11,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133031133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:11,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133031133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:11,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T06:16:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T06:16:11,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T06:16:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/b09d4e92b44c4c3482ee3dc5209c8f15 2024-11-09T06:16:11,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T06:16:11,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:11,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/d30d9a4108024a20bdb319563c8c3870 is 50, key is test_row_0/C:col10/1731132969290/Put/seqid=0 2024-11-09T06:16:11,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133031512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:11,523 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_0' in f48863a85e472c28d23fb8419a1993ee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:16:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:11,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741979_1155 (size=12151) 2024-11-09T06:16:11,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T06:16:11,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:11,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133031643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:11,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133031644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:11,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133031644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133031644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:11,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T06:16:11,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:11,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T06:16:11,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T06:16:11,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:11,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:11,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/d30d9a4108024a20bdb319563c8c3870 2024-11-09T06:16:11,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/f8dc3aeffb9545a3bf122452ef838532 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f8dc3aeffb9545a3bf122452ef838532 2024-11-09T06:16:11,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f8dc3aeffb9545a3bf122452ef838532, entries=200, sequenceid=202, filesize=38.8 K 2024-11-09T06:16:11,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/b09d4e92b44c4c3482ee3dc5209c8f15 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/b09d4e92b44c4c3482ee3dc5209c8f15 2024-11-09T06:16:11,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/b09d4e92b44c4c3482ee3dc5209c8f15, entries=150, sequenceid=202, filesize=11.9 K 2024-11-09T06:16:11,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/d30d9a4108024a20bdb319563c8c3870 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/d30d9a4108024a20bdb319563c8c3870 2024-11-09T06:16:11,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/d30d9a4108024a20bdb319563c8c3870, entries=150, sequenceid=202, filesize=11.9 K 2024-11-09T06:16:11,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~431.81 KB/442176, currentSize=33.54 KB/34350 for f48863a85e472c28d23fb8419a1993ee in 1524ms, sequenceid=202, compaction requested=true 2024-11-09T06:16:11,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:11,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:11,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:11,994 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:11,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:11,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:11,994 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:11,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:11,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:11,995 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:11,995 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/B is initiating minor compaction (all files) 2024-11-09T06:16:11,995 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/B in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,996 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/af8cdc71e7d5423eb88b0ff2023cd26c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a108e19c5d33404091c2b75bb234138e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/b09d4e92b44c4c3482ee3dc5209c8f15] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=35.8 K 2024-11-09T06:16:11,996 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102165 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:11,996 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/A is initiating minor compaction (all files) 2024-11-09T06:16:11,996 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/A in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,996 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/b179bc44e54443f49b22861632d3bff9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48e7f92988484ee6bb6a5d0955742db9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f8dc3aeffb9545a3bf122452ef838532] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=99.8 K 2024-11-09T06:16:11,996 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:11,996 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/b179bc44e54443f49b22861632d3bff9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48e7f92988484ee6bb6a5d0955742db9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f8dc3aeffb9545a3bf122452ef838532] 2024-11-09T06:16:11,997 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting af8cdc71e7d5423eb88b0ff2023cd26c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731132967845 2024-11-09T06:16:11,997 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b179bc44e54443f49b22861632d3bff9, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731132967845 2024-11-09T06:16:11,997 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a108e19c5d33404091c2b75bb234138e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731132967943 2024-11-09T06:16:11,998 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48e7f92988484ee6bb6a5d0955742db9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731132967943 2024-11-09T06:16:11,998 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b09d4e92b44c4c3482ee3dc5209c8f15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731132969289 2024-11-09T06:16:11,999 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8dc3aeffb9545a3bf122452ef838532, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731132969277 2024-11-09T06:16:12,013 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:12,015 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:12,016 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/a81786aaea954958af235a628e08fa39 is 50, key is test_row_0/B:col10/1731132969290/Put/seqid=0 2024-11-09T06:16:12,019 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110911c4f3ada21449aba9cce613be6dd295_f48863a85e472c28d23fb8419a1993ee store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:12,021 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110911c4f3ada21449aba9cce613be6dd295_f48863a85e472c28d23fb8419a1993ee, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:12,021 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110911c4f3ada21449aba9cce613be6dd295_f48863a85e472c28d23fb8419a1993ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:12,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741980_1156 (size=12459) 2024-11-09T06:16:12,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741981_1157 (size=4469) 2024-11-09T06:16:12,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T06:16:12,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:12,108 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T06:16:12,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:12,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:12,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:12,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:12,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:12,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:12,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109df5d2fbe971a48a9880c823ba0f6f1cc_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132970489/Put/seqid=0 2024-11-09T06:16:12,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741982_1158 (size=12304) 2024-11-09T06:16:12,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:12,135 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109df5d2fbe971a48a9880c823ba0f6f1cc_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109df5d2fbe971a48a9880c823ba0f6f1cc_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:12,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/8d741e2e4265414296b801f02c3d69da, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:12,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/8d741e2e4265414296b801f02c3d69da is 175, key is test_row_0/A:col10/1731132970489/Put/seqid=0 2024-11-09T06:16:12,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741983_1159 (size=31105) 2024-11-09T06:16:12,454 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/a81786aaea954958af235a628e08fa39 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a81786aaea954958af235a628e08fa39 2024-11-09T06:16:12,461 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#A#compaction#131 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:12,463 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/a84a7eedf5ab406faaa564ab6ec0e0f7 is 175, key is test_row_0/A:col10/1731132969290/Put/seqid=0 2024-11-09T06:16:12,473 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/B of f48863a85e472c28d23fb8419a1993ee into a81786aaea954958af235a628e08fa39(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:12,473 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:12,473 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/B, priority=13, startTime=1731132971994; duration=0sec 2024-11-09T06:16:12,474 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:12,474 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:B 2024-11-09T06:16:12,474 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:16:12,476 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:16:12,476 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/C is initiating minor compaction (all files) 2024-11-09T06:16:12,476 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/C in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:12,476 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/1b357a37af844b68af29b85261bcb5f9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dcc6ae97728a4743be5090f5ab9925de, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/7153a6a1539746009e8250018772f713, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/d30d9a4108024a20bdb319563c8c3870] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=47.5 K 2024-11-09T06:16:12,480 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b357a37af844b68af29b85261bcb5f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731132966827 2024-11-09T06:16:12,480 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting dcc6ae97728a4743be5090f5ab9925de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731132967845 2024-11-09T06:16:12,481 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7153a6a1539746009e8250018772f713, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731132967943 2024-11-09T06:16:12,484 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting d30d9a4108024a20bdb319563c8c3870, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731132969289 2024-11-09T06:16:12,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741984_1160 (size=31413) 2024-11-09T06:16:12,533 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#C#compaction#134 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:12,534 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/326da85f7fa14260bb414a7eb84f1d33 is 50, key is test_row_0/C:col10/1731132969290/Put/seqid=0 2024-11-09T06:16:12,547 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/8d741e2e4265414296b801f02c3d69da 2024-11-09T06:16:12,548 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/a84a7eedf5ab406faaa564ab6ec0e0f7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/a84a7eedf5ab406faaa564ab6ec0e0f7 2024-11-09T06:16:12,563 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/A of f48863a85e472c28d23fb8419a1993ee into a84a7eedf5ab406faaa564ab6ec0e0f7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:12,564 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:12,564 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/A, priority=13, startTime=1731132971994; duration=0sec 2024-11-09T06:16:12,564 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:12,564 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:A 2024-11-09T06:16:12,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/fa6f38078eca4554840da66186de0e01 is 50, key is test_row_0/B:col10/1731132970489/Put/seqid=0 2024-11-09T06:16:12,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741985_1161 (size=12425) 2024-11-09T06:16:12,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741986_1162 (size=12151) 2024-11-09T06:16:12,622 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/fa6f38078eca4554840da66186de0e01 2024-11-09T06:16:12,630 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/326da85f7fa14260bb414a7eb84f1d33 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/326da85f7fa14260bb414a7eb84f1d33 2024-11-09T06:16:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/71ba62b4310d4877a564736a8d37e652 is 50, key is test_row_0/C:col10/1731132970489/Put/seqid=0 2024-11-09T06:16:12,641 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/C of f48863a85e472c28d23fb8419a1993ee into 326da85f7fa14260bb414a7eb84f1d33(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:12,641 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:12,641 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/C, priority=12, startTime=1731132971994; duration=0sec 2024-11-09T06:16:12,641 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:12,641 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:C 2024-11-09T06:16:12,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:12,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741987_1163 (size=12151) 2024-11-09T06:16:12,686 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/71ba62b4310d4877a564736a8d37e652 2024-11-09T06:16:12,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/8d741e2e4265414296b801f02c3d69da as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/8d741e2e4265414296b801f02c3d69da 2024-11-09T06:16:12,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731132972684, max timestamp=1731132972718Num uniques 859; 2024-11-09T06:16:12,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:12,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731132972684, max timestamp=1731132972718Num uniques 859; 2024-11-09T06:16:12,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:12,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731132972684, max timestamp=1731132972719Num uniques 859; 2024-11-09T06:16:12,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:12,724 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/8d741e2e4265414296b801f02c3d69da, entries=150, sequenceid=210, filesize=30.4 K 2024-11-09T06:16:12,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/fa6f38078eca4554840da66186de0e01 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fa6f38078eca4554840da66186de0e01 2024-11-09T06:16:12,741 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fa6f38078eca4554840da66186de0e01, entries=150, sequenceid=210, filesize=11.9 K 2024-11-09T06:16:12,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/71ba62b4310d4877a564736a8d37e652 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/71ba62b4310d4877a564736a8d37e652 2024-11-09T06:16:12,757 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/71ba62b4310d4877a564736a8d37e652, entries=150, sequenceid=210, filesize=11.9 K 2024-11-09T06:16:12,760 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=100.63 KB/103050 for f48863a85e472c28d23fb8419a1993ee in 652ms, sequenceid=210, compaction requested=false 2024-11-09T06:16:12,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:12,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:12,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-09T06:16:12,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=107.34 KB heapSize=295.92 KB 2024-11-09T06:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-09T06:16:12,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:12,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:12,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:12,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:12,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:12,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:12,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-09T06:16:12,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0610 sec 2024-11-09T06:16:12,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.0730 sec 2024-11-09T06:16:12,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109685d04882a0c4ae8b92d00d494c62dff_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132972752/Put/seqid=0 2024-11-09T06:16:12,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133032823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T06:16:12,857 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:12,858 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-09T06:16:12,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133032827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133032828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:12,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133032829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T06:16:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:12,863 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:12,864 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:12,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:12,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741988_1164 (size=22268) 2024-11-09T06:16:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133032963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T06:16:12,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133032973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:12,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133032973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:12,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133032973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:13,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T06:16:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133033174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:13,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:13,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T06:16:13,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:13,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T06:16:13,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133033182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:13,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133033183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:13,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133033183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:13,267 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:13,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109685d04882a0c4ae8b92d00d494c62dff_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109685d04882a0c4ae8b92d00d494c62dff_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:13,280 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/3b9efa401c3e4a97be4f292222b8d152, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:13,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/3b9efa401c3e4a97be4f292222b8d152 is 175, key is test_row_0/A:col10/1731132972752/Put/seqid=0 2024-11-09T06:16:13,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741989_1165 (size=65673) 2024-11-09T06:16:13,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T06:16:13,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:13,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133033483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:13,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T06:16:13,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:13,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T06:16:13,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133033493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:13,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133033494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:13,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133033495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:13,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133033532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:13,535 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_0' in f48863a85e472c28d23fb8419a1993ee of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:16:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:13,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T06:16:13,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:13,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/3b9efa401c3e4a97be4f292222b8d152 2024-11-09T06:16:13,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/32fcf8fb928646a99f643cc058d88308 is 50, key is test_row_0/B:col10/1731132972752/Put/seqid=0 2024-11-09T06:16:13,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741990_1166 (size=12151) 2024-11-09T06:16:13,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/32fcf8fb928646a99f643cc058d88308 2024-11-09T06:16:13,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/6bd7c3c861e14c11b1ce27961ede3ee5 is 50, key is test_row_0/C:col10/1731132972752/Put/seqid=0 2024-11-09T06:16:13,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741991_1167 (size=12151) 2024-11-09T06:16:13,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/6bd7c3c861e14c11b1ce27961ede3ee5 2024-11-09T06:16:13,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/3b9efa401c3e4a97be4f292222b8d152 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/3b9efa401c3e4a97be4f292222b8d152 2024-11-09T06:16:13,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/3b9efa401c3e4a97be4f292222b8d152, entries=350, sequenceid=244, filesize=64.1 K 2024-11-09T06:16:13,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/32fcf8fb928646a99f643cc058d88308 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/32fcf8fb928646a99f643cc058d88308 2024-11-09T06:16:13,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T06:16:13,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:13,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:13,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/32fcf8fb928646a99f643cc058d88308, entries=150, sequenceid=244, filesize=11.9 K 2024-11-09T06:16:13,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/6bd7c3c861e14c11b1ce27961ede3ee5 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bd7c3c861e14c11b1ce27961ede3ee5 2024-11-09T06:16:13,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bd7c3c861e14c11b1ce27961ede3ee5, entries=150, sequenceid=244, filesize=11.9 K 2024-11-09T06:16:13,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~295.88 KB/302976, currentSize=73.80 KB/75570 for f48863a85e472c28d23fb8419a1993ee in 1078ms, sequenceid=244, compaction requested=true 2024-11-09T06:16:13,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:13,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:13,840 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:13,840 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:13,841 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:13,841 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:13,841 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/B is initiating minor compaction (all files) 2024-11-09T06:16:13,841 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/A is initiating minor compaction (all files) 2024-11-09T06:16:13,842 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/B in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,842 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a81786aaea954958af235a628e08fa39, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fa6f38078eca4554840da66186de0e01, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/32fcf8fb928646a99f643cc058d88308] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=35.9 K 2024-11-09T06:16:13,842 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/A in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,842 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/a84a7eedf5ab406faaa564ab6ec0e0f7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/8d741e2e4265414296b801f02c3d69da, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/3b9efa401c3e4a97be4f292222b8d152] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=125.2 K 2024-11-09T06:16:13,842 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,842 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/a84a7eedf5ab406faaa564ab6ec0e0f7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/8d741e2e4265414296b801f02c3d69da, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/3b9efa401c3e4a97be4f292222b8d152] 2024-11-09T06:16:13,842 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a81786aaea954958af235a628e08fa39, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731132969289 2024-11-09T06:16:13,843 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting fa6f38078eca4554840da66186de0e01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731132970487 2024-11-09T06:16:13,843 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a84a7eedf5ab406faaa564ab6ec0e0f7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731132969289 2024-11-09T06:16:13,844 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d741e2e4265414296b801f02c3d69da, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731132970487 2024-11-09T06:16:13,845 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 32fcf8fb928646a99f643cc058d88308, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731132972752 2024-11-09T06:16:13,845 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b9efa401c3e4a97be4f292222b8d152, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731132972736 2024-11-09T06:16:13,862 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:13,873 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110959be3d88e23b402d9226d4d5fcc360df_f48863a85e472c28d23fb8419a1993ee store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:13,875 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#B#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:13,875 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/16ebe8d02ae0454fa6c618e81427e32d is 50, key is test_row_0/B:col10/1731132972752/Put/seqid=0 2024-11-09T06:16:13,875 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110959be3d88e23b402d9226d4d5fcc360df_f48863a85e472c28d23fb8419a1993ee, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:13,876 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110959be3d88e23b402d9226d4d5fcc360df_f48863a85e472c28d23fb8419a1993ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:13,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741993_1169 (size=12561) 2024-11-09T06:16:13,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741992_1168 (size=4469) 2024-11-09T06:16:13,910 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#A#compaction#140 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:13,911 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/cf26e2350ccf4818bd54529c95b2f6dd is 175, key is test_row_0/A:col10/1731132972752/Put/seqid=0 2024-11-09T06:16:13,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741994_1170 (size=31515) 2024-11-09T06:16:13,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T06:16:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,958 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T06:16:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:13,973 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/cf26e2350ccf4818bd54529c95b2f6dd as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/cf26e2350ccf4818bd54529c95b2f6dd 2024-11-09T06:16:13,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098d5a69563c564298a3362d46ceddfee0_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132972796/Put/seqid=0 2024-11-09T06:16:13,991 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/A of f48863a85e472c28d23fb8419a1993ee into cf26e2350ccf4818bd54529c95b2f6dd(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:13,991 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:13,991 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/A, priority=13, startTime=1731132973839; duration=0sec 2024-11-09T06:16:13,992 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:13,992 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:A 2024-11-09T06:16:13,992 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:13,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:13,996 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:13,997 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/C is initiating minor compaction (all files) 2024-11-09T06:16:13,997 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/C in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:13,997 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/326da85f7fa14260bb414a7eb84f1d33, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/71ba62b4310d4877a564736a8d37e652, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bd7c3c861e14c11b1ce27961ede3ee5] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=35.9 K 2024-11-09T06:16:13,998 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 326da85f7fa14260bb414a7eb84f1d33, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731132969289 2024-11-09T06:16:13,999 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71ba62b4310d4877a564736a8d37e652, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731132970487 2024-11-09T06:16:13,999 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bd7c3c861e14c11b1ce27961ede3ee5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731132972752 2024-11-09T06:16:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T06:16:14,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741995_1171 (size=12354) 2024-11-09T06:16:14,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:14,017 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098d5a69563c564298a3362d46ceddfee0_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098d5a69563c564298a3362d46ceddfee0_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/c4905c3443b243b6a39336501f70ede9, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:14,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/c4905c3443b243b6a39336501f70ede9 is 175, key is test_row_0/A:col10/1731132972796/Put/seqid=0 2024-11-09T06:16:14,028 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#C#compaction#143 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:14,029 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/e2d64f49327d4a65b7f64ac6143adbb3 is 50, key is test_row_0/C:col10/1731132972752/Put/seqid=0 2024-11-09T06:16:14,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741996_1172 (size=31155) 2024-11-09T06:16:14,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741997_1173 (size=12527) 2024-11-09T06:16:14,104 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/e2d64f49327d4a65b7f64ac6143adbb3 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/e2d64f49327d4a65b7f64ac6143adbb3 2024-11-09T06:16:14,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133034089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133034106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,114 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/C of f48863a85e472c28d23fb8419a1993ee into e2d64f49327d4a65b7f64ac6143adbb3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:14,114 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:14,114 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/C, priority=13, startTime=1731132973840; duration=0sec 2024-11-09T06:16:14,114 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:14,114 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:C 2024-11-09T06:16:14,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133034109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133034109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133034214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,221 INFO [master/4a84bcce7bad:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-09T06:16:14,221 INFO [master/4a84bcce7bad:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-09T06:16:14,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133034223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133034223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133034223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,296 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/16ebe8d02ae0454fa6c618e81427e32d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/16ebe8d02ae0454fa6c618e81427e32d 2024-11-09T06:16:14,303 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/B of f48863a85e472c28d23fb8419a1993ee into 16ebe8d02ae0454fa6c618e81427e32d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:14,303 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:14,303 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/B, priority=13, startTime=1731132973840; duration=0sec 2024-11-09T06:16:14,304 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:14,304 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:B 2024-11-09T06:16:14,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133034422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133034433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133034433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133034434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,467 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/c4905c3443b243b6a39336501f70ede9 2024-11-09T06:16:14,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/def9943d9b8443d18860780d14bffb27 is 50, key is test_row_0/B:col10/1731132972796/Put/seqid=0 2024-11-09T06:16:14,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741998_1174 (size=12201) 2024-11-09T06:16:14,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133034735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133034744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133034744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133034744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:14,970 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/def9943d9b8443d18860780d14bffb27 2024-11-09T06:16:14,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/8b6e02be9fcc41f9ba27eee9f9c7c319 is 50, key is test_row_0/C:col10/1731132972796/Put/seqid=0 2024-11-09T06:16:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T06:16:15,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741999_1175 (size=12201) 2024-11-09T06:16:15,022 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/8b6e02be9fcc41f9ba27eee9f9c7c319 2024-11-09T06:16:15,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/c4905c3443b243b6a39336501f70ede9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c4905c3443b243b6a39336501f70ede9 2024-11-09T06:16:15,041 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c4905c3443b243b6a39336501f70ede9, entries=150, sequenceid=258, filesize=30.4 K 2024-11-09T06:16:15,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/def9943d9b8443d18860780d14bffb27 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/def9943d9b8443d18860780d14bffb27 2024-11-09T06:16:15,057 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/def9943d9b8443d18860780d14bffb27, entries=150, sequenceid=258, filesize=11.9 K 2024-11-09T06:16:15,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/8b6e02be9fcc41f9ba27eee9f9c7c319 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8b6e02be9fcc41f9ba27eee9f9c7c319 2024-11-09T06:16:15,070 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8b6e02be9fcc41f9ba27eee9f9c7c319, entries=150, sequenceid=258, filesize=11.9 K 2024-11-09T06:16:15,072 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for f48863a85e472c28d23fb8419a1993ee in 1114ms, sequenceid=258, compaction requested=false 2024-11-09T06:16:15,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:15,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:15,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-09T06:16:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-09T06:16:15,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-09T06:16:15,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2130 sec 2024-11-09T06:16:15,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.2230 sec 2024-11-09T06:16:15,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:16:15,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:15,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:15,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:15,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:15,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:15,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:15,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a6a6c4c4811f4769b8b2f11507b3a3a3_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132974103/Put/seqid=0 2024-11-09T06:16:15,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133035310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133035311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742000_1176 (size=14994) 2024-11-09T06:16:15,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133035320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133035322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133035433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133035434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133035444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133035445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133035653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133035654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133035664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133035664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,728 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:15,740 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a6a6c4c4811f4769b8b2f11507b3a3a3_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a6a6c4c4811f4769b8b2f11507b3a3a3_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:15,742 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/354585acbaff4e4e8647539cfa320de6, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:15,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/354585acbaff4e4e8647539cfa320de6 is 175, key is test_row_0/A:col10/1731132974103/Put/seqid=0 2024-11-09T06:16:15,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742001_1177 (size=39949) 2024-11-09T06:16:15,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133035962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133035963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133035983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133035983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:15,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:15,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:16,183 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/354585acbaff4e4e8647539cfa320de6 2024-11-09T06:16:16,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/3d7068c90f3248278300c63495accb61 is 50, key is test_row_0/B:col10/1731132974103/Put/seqid=0 2024-11-09T06:16:16,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742002_1178 (size=12301) 2024-11-09T06:16:16,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133036472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:16,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133036483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:16,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133036496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:16,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133036496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:16,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/3d7068c90f3248278300c63495accb61 2024-11-09T06:16:16,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/6bdd81a40e894c3587317045946aeaf1 is 50, key is test_row_0/C:col10/1731132974103/Put/seqid=0 2024-11-09T06:16:16,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742003_1179 (size=12301) 2024-11-09T06:16:16,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/6bdd81a40e894c3587317045946aeaf1 2024-11-09T06:16:16,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/354585acbaff4e4e8647539cfa320de6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/354585acbaff4e4e8647539cfa320de6 2024-11-09T06:16:16,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/354585acbaff4e4e8647539cfa320de6, entries=200, sequenceid=281, filesize=39.0 K 2024-11-09T06:16:16,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/3d7068c90f3248278300c63495accb61 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/3d7068c90f3248278300c63495accb61 2024-11-09T06:16:16,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/3d7068c90f3248278300c63495accb61, entries=150, sequenceid=281, filesize=12.0 K 2024-11-09T06:16:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/6bdd81a40e894c3587317045946aeaf1 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bdd81a40e894c3587317045946aeaf1 2024-11-09T06:16:16,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bdd81a40e894c3587317045946aeaf1, entries=150, sequenceid=281, filesize=12.0 K 2024-11-09T06:16:16,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f48863a85e472c28d23fb8419a1993ee in 1438ms, sequenceid=281, compaction requested=true 2024-11-09T06:16:16,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:16,702 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:16,703 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:16,703 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/A is initiating minor compaction (all files) 2024-11-09T06:16:16,703 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/A in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:16,703 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/cf26e2350ccf4818bd54529c95b2f6dd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c4905c3443b243b6a39336501f70ede9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/354585acbaff4e4e8647539cfa320de6] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=100.2 K 2024-11-09T06:16:16,703 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:16,703 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/cf26e2350ccf4818bd54529c95b2f6dd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c4905c3443b243b6a39336501f70ede9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/354585acbaff4e4e8647539cfa320de6] 2024-11-09T06:16:16,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:16,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:16,704 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf26e2350ccf4818bd54529c95b2f6dd, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731132972752 2024-11-09T06:16:16,704 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:16,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:16,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:16,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:16,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:16,705 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4905c3443b243b6a39336501f70ede9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731132972778 2024-11-09T06:16:16,705 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 354585acbaff4e4e8647539cfa320de6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731132974088 2024-11-09T06:16:16,706 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37063 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:16,706 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/B is initiating minor compaction (all files) 2024-11-09T06:16:16,706 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/B in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:16,706 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/16ebe8d02ae0454fa6c618e81427e32d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/def9943d9b8443d18860780d14bffb27, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/3d7068c90f3248278300c63495accb61] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=36.2 K 2024-11-09T06:16:16,707 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 16ebe8d02ae0454fa6c618e81427e32d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731132972752 2024-11-09T06:16:16,707 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting def9943d9b8443d18860780d14bffb27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731132972778 2024-11-09T06:16:16,708 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d7068c90f3248278300c63495accb61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731132974088 2024-11-09T06:16:16,718 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:16,723 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:16,723 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/8a9a22ce4d5249d29ba8eab4c6e58694 is 50, key is test_row_0/B:col10/1731132974103/Put/seqid=0 2024-11-09T06:16:16,729 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110905e54c48b50247d7bdc0eb3dfb6dc758_f48863a85e472c28d23fb8419a1993ee store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:16,731 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110905e54c48b50247d7bdc0eb3dfb6dc758_f48863a85e472c28d23fb8419a1993ee, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:16,732 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110905e54c48b50247d7bdc0eb3dfb6dc758_f48863a85e472c28d23fb8419a1993ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:16,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742004_1180 (size=12813) 2024-11-09T06:16:16,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742005_1181 (size=4469) 2024-11-09T06:16:16,812 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#A#compaction#149 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:16,813 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/6b08ff941adb494db7a1784a17d1d64f is 175, key is test_row_0/A:col10/1731132974103/Put/seqid=0 2024-11-09T06:16:16,840 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/8a9a22ce4d5249d29ba8eab4c6e58694 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/8a9a22ce4d5249d29ba8eab4c6e58694 2024-11-09T06:16:16,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742006_1182 (size=31767) 2024-11-09T06:16:16,866 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/B of f48863a85e472c28d23fb8419a1993ee into 8a9a22ce4d5249d29ba8eab4c6e58694(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:16,866 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:16,866 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/B, priority=13, startTime=1731132976704; duration=0sec 2024-11-09T06:16:16,866 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:16,866 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:B 2024-11-09T06:16:16,866 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:16,867 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/6b08ff941adb494db7a1784a17d1d64f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/6b08ff941adb494db7a1784a17d1d64f 2024-11-09T06:16:16,873 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37029 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:16,873 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/C is initiating minor compaction (all files) 2024-11-09T06:16:16,873 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/C in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:16,873 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/e2d64f49327d4a65b7f64ac6143adbb3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8b6e02be9fcc41f9ba27eee9f9c7c319, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bdd81a40e894c3587317045946aeaf1] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=36.2 K 2024-11-09T06:16:16,875 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d64f49327d4a65b7f64ac6143adbb3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731132972752 2024-11-09T06:16:16,876 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b6e02be9fcc41f9ba27eee9f9c7c319, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731132972778 2024-11-09T06:16:16,879 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bdd81a40e894c3587317045946aeaf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731132974088 2024-11-09T06:16:16,894 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/A of f48863a85e472c28d23fb8419a1993ee into 6b08ff941adb494db7a1784a17d1d64f(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:16,894 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:16,894 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/A, priority=13, startTime=1731132976701; duration=0sec 2024-11-09T06:16:16,894 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:16,894 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:A 2024-11-09T06:16:16,921 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#C#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:16,921 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/230f77b9c9ac4e1084327f09dc98c89c is 50, key is test_row_0/C:col10/1731132974103/Put/seqid=0 2024-11-09T06:16:16,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742007_1183 (size=12779) 2024-11-09T06:16:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T06:16:17,043 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:17,044 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-09T06:16:17,046 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:17,047 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:17,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T06:16:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T06:16:17,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T06:16:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:17,201 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:16:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:17,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095ad6cb31552545c4a6eb9578cfab6620_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132975312/Put/seqid=0 2024-11-09T06:16:17,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742008_1184 (size=12454) 2024-11-09T06:16:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T06:16:17,412 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/230f77b9c9ac4e1084327f09dc98c89c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/230f77b9c9ac4e1084327f09dc98c89c 2024-11-09T06:16:17,424 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/C of f48863a85e472c28d23fb8419a1993ee into 230f77b9c9ac4e1084327f09dc98c89c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:17,424 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:17,424 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/C, priority=13, startTime=1731132976704; duration=0sec 2024-11-09T06:16:17,424 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:17,424 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:C 2024-11-09T06:16:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:17,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:17,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133037565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133037569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133037571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133037573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133037574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:17,668 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095ad6cb31552545c4a6eb9578cfab6620_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095ad6cb31552545c4a6eb9578cfab6620_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T06:16:17,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/4a7c0d8f42874609ae1496abc709d64a, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:17,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/4a7c0d8f42874609ae1496abc709d64a is 175, key is test_row_0/A:col10/1731132975312/Put/seqid=0 2024-11-09T06:16:17,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133037683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133037683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133037683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133037683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133037683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742009_1185 (size=31255) 2024-11-09T06:16:17,721 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/4a7c0d8f42874609ae1496abc709d64a 2024-11-09T06:16:17,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/76cff67c97cc4986bfc5599c0380d19f is 50, key is test_row_0/B:col10/1731132975312/Put/seqid=0 2024-11-09T06:16:17,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742010_1186 (size=12301) 2024-11-09T06:16:17,777 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/76cff67c97cc4986bfc5599c0380d19f 2024-11-09T06:16:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/9afd130693b648aa83fbc58f62fdaa76 is 50, key is test_row_0/C:col10/1731132975312/Put/seqid=0 2024-11-09T06:16:17,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742011_1187 (size=12301) 2024-11-09T06:16:17,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133037892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133037892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133037893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133037894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133037894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T06:16:18,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133038203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133038203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133038203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133038203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133038203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,223 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/9afd130693b648aa83fbc58f62fdaa76 2024-11-09T06:16:18,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/4a7c0d8f42874609ae1496abc709d64a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/4a7c0d8f42874609ae1496abc709d64a 2024-11-09T06:16:18,242 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/4a7c0d8f42874609ae1496abc709d64a, entries=150, sequenceid=296, filesize=30.5 K 2024-11-09T06:16:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/76cff67c97cc4986bfc5599c0380d19f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/76cff67c97cc4986bfc5599c0380d19f 2024-11-09T06:16:18,257 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/76cff67c97cc4986bfc5599c0380d19f, entries=150, sequenceid=296, filesize=12.0 K 2024-11-09T06:16:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/9afd130693b648aa83fbc58f62fdaa76 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/9afd130693b648aa83fbc58f62fdaa76 2024-11-09T06:16:18,271 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/9afd130693b648aa83fbc58f62fdaa76, entries=150, sequenceid=296, filesize=12.0 K 2024-11-09T06:16:18,272 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f48863a85e472c28d23fb8419a1993ee in 1071ms, sequenceid=296, compaction requested=false 2024-11-09T06:16:18,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:18,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:18,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-09T06:16:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-09T06:16:18,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-09T06:16:18,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2270 sec 2024-11-09T06:16:18,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.2340 sec 2024-11-09T06:16:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:18,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:16:18,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:18,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:18,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:18,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:18,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:18,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:18,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e6bde1c90e8e4d4e8651352f335b9e2e_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132977572/Put/seqid=0 2024-11-09T06:16:18,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742012_1188 (size=12454) 2024-11-09T06:16:18,757 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:18,762 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e6bde1c90e8e4d4e8651352f335b9e2e_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e6bde1c90e8e4d4e8651352f335b9e2e_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:18,763 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/9d719e9c2cdb4094a50e936dedd015bb, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:18,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/9d719e9c2cdb4094a50e936dedd015bb is 175, key is test_row_0/A:col10/1731132977572/Put/seqid=0 2024-11-09T06:16:18,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133038768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133038768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133038768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133038770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133038777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742013_1189 (size=31255) 2024-11-09T06:16:18,793 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/9d719e9c2cdb4094a50e936dedd015bb 2024-11-09T06:16:18,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/5ef83a2fb59a48b18fd88944d32c6ed7 is 50, key is test_row_0/B:col10/1731132977572/Put/seqid=0 2024-11-09T06:16:18,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742014_1190 (size=12301) 2024-11-09T06:16:18,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/5ef83a2fb59a48b18fd88944d32c6ed7 2024-11-09T06:16:18,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/b61a368943654ce59a232dfda1db034f is 50, key is test_row_0/C:col10/1731132977572/Put/seqid=0 2024-11-09T06:16:18,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742015_1191 (size=12301) 2024-11-09T06:16:18,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/b61a368943654ce59a232dfda1db034f 2024-11-09T06:16:18,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/9d719e9c2cdb4094a50e936dedd015bb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/9d719e9c2cdb4094a50e936dedd015bb 2024-11-09T06:16:18,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/9d719e9c2cdb4094a50e936dedd015bb, entries=150, sequenceid=318, filesize=30.5 K 2024-11-09T06:16:18,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/5ef83a2fb59a48b18fd88944d32c6ed7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/5ef83a2fb59a48b18fd88944d32c6ed7 2024-11-09T06:16:18,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133038894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133038894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133038894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133038894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133038894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:18,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/5ef83a2fb59a48b18fd88944d32c6ed7, entries=150, sequenceid=318, filesize=12.0 K 2024-11-09T06:16:18,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/b61a368943654ce59a232dfda1db034f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/b61a368943654ce59a232dfda1db034f 2024-11-09T06:16:18,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/b61a368943654ce59a232dfda1db034f, entries=150, sequenceid=318, filesize=12.0 K 2024-11-09T06:16:18,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for f48863a85e472c28d23fb8419a1993ee in 194ms, sequenceid=318, compaction requested=true 2024-11-09T06:16:18,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:18,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:18,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:18,910 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:18,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:18,910 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:18,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:18,912 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:18,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:18,912 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/A is initiating minor compaction (all files) 2024-11-09T06:16:18,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:18,912 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:18,912 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/A in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:18,912 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/B is initiating minor compaction (all files) 2024-11-09T06:16:18,912 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/B in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:18,912 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/6b08ff941adb494db7a1784a17d1d64f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/4a7c0d8f42874609ae1496abc709d64a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/9d719e9c2cdb4094a50e936dedd015bb] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=92.1 K 2024-11-09T06:16:18,912 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/8a9a22ce4d5249d29ba8eab4c6e58694, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/76cff67c97cc4986bfc5599c0380d19f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/5ef83a2fb59a48b18fd88944d32c6ed7] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=36.5 K 2024-11-09T06:16:18,912 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:18,912 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/6b08ff941adb494db7a1784a17d1d64f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/4a7c0d8f42874609ae1496abc709d64a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/9d719e9c2cdb4094a50e936dedd015bb] 2024-11-09T06:16:18,913 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b08ff941adb494db7a1784a17d1d64f, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731132974088 2024-11-09T06:16:18,913 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a9a22ce4d5249d29ba8eab4c6e58694, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731132974088 2024-11-09T06:16:18,914 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a7c0d8f42874609ae1496abc709d64a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731132975275 2024-11-09T06:16:18,914 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 76cff67c97cc4986bfc5599c0380d19f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731132975275 2024-11-09T06:16:18,914 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d719e9c2cdb4094a50e936dedd015bb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731132977572 2024-11-09T06:16:18,914 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ef83a2fb59a48b18fd88944d32c6ed7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731132977572 2024-11-09T06:16:18,939 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#B#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:18,939 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/87a276bc2b624d6a93455d706fe60d69 is 50, key is test_row_0/B:col10/1731132977572/Put/seqid=0 2024-11-09T06:16:18,941 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:18,947 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109e0b5311dd7a74a38aaf76f1ae76d3488_f48863a85e472c28d23fb8419a1993ee store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:18,949 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109e0b5311dd7a74a38aaf76f1ae76d3488_f48863a85e472c28d23fb8419a1993ee, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:18,950 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e0b5311dd7a74a38aaf76f1ae76d3488_f48863a85e472c28d23fb8419a1993ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:18,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742016_1192 (size=12915) 2024-11-09T06:16:19,000 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/87a276bc2b624d6a93455d706fe60d69 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/87a276bc2b624d6a93455d706fe60d69 2024-11-09T06:16:19,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742017_1193 (size=4469) 2024-11-09T06:16:19,004 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#A#compaction#159 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:19,004 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/41b5ddfcbd3f4fc689a0686987a272a6 is 175, key is test_row_0/A:col10/1731132977572/Put/seqid=0 2024-11-09T06:16:19,013 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/B of f48863a85e472c28d23fb8419a1993ee into 87a276bc2b624d6a93455d706fe60d69(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:19,013 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:19,013 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/B, priority=13, startTime=1731132978910; duration=0sec 2024-11-09T06:16:19,013 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:19,014 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:B 2024-11-09T06:16:19,014 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:19,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742018_1194 (size=31869) 2024-11-09T06:16:19,017 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:19,017 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/C is initiating minor compaction (all files) 2024-11-09T06:16:19,017 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/C in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:19,017 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/230f77b9c9ac4e1084327f09dc98c89c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/9afd130693b648aa83fbc58f62fdaa76, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/b61a368943654ce59a232dfda1db034f] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=36.5 K 2024-11-09T06:16:19,018 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 230f77b9c9ac4e1084327f09dc98c89c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731132974088 2024-11-09T06:16:19,018 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9afd130693b648aa83fbc58f62fdaa76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731132975275 2024-11-09T06:16:19,019 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b61a368943654ce59a232dfda1db034f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731132977572 2024-11-09T06:16:19,025 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/41b5ddfcbd3f4fc689a0686987a272a6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/41b5ddfcbd3f4fc689a0686987a272a6 2024-11-09T06:16:19,041 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/A of f48863a85e472c28d23fb8419a1993ee into 41b5ddfcbd3f4fc689a0686987a272a6(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:19,041 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:19,042 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/A, priority=13, startTime=1731132978910; duration=0sec 2024-11-09T06:16:19,042 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:19,042 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:A 2024-11-09T06:16:19,056 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#C#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:19,056 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/20b15da6eb204be7bd15ea39e7f12478 is 50, key is test_row_0/C:col10/1731132977572/Put/seqid=0 2024-11-09T06:16:19,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742019_1195 (size=12881) 2024-11-09T06:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:19,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T06:16:19,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:19,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:19,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:19,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:19,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:19,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:19,130 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/20b15da6eb204be7bd15ea39e7f12478 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/20b15da6eb204be7bd15ea39e7f12478 2024-11-09T06:16:19,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109442a320377d14d09be16f9e0755d43df_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132979117/Put/seqid=0 2024-11-09T06:16:19,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742020_1196 (size=17534) 2024-11-09T06:16:19,140 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/C of f48863a85e472c28d23fb8419a1993ee into 20b15da6eb204be7bd15ea39e7f12478(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:19,140 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:19,140 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/C, priority=13, startTime=1731132978910; duration=0sec 2024-11-09T06:16:19,140 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:19,140 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:C 2024-11-09T06:16:19,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133039158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133039162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133039168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133039169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133039171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T06:16:19,195 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:19,195 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-09T06:16:19,199 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T06:16:19,200 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:19,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:19,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133039273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133039275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133039283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133039283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133039283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T06:16:19,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T06:16:19,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:19,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:19,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:19,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133039483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133039483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133039493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133039493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133039503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T06:16:19,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:19,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:19,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:19,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T06:16:19,540 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:19,551 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109442a320377d14d09be16f9e0755d43df_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109442a320377d14d09be16f9e0755d43df_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:19,553 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/ac495e2ee18148ce89f451305a3418a8, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:19,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/ac495e2ee18148ce89f451305a3418a8 is 175, key is test_row_0/A:col10/1731132979117/Put/seqid=0 2024-11-09T06:16:19,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742021_1197 (size=48639) 2024-11-09T06:16:19,587 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/ac495e2ee18148ce89f451305a3418a8 2024-11-09T06:16:19,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/1a79b4e768c04e5992641ee30e6c465f is 50, key is test_row_0/B:col10/1731132979117/Put/seqid=0 2024-11-09T06:16:19,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742022_1198 (size=12301) 2024-11-09T06:16:19,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/1a79b4e768c04e5992641ee30e6c465f 2024-11-09T06:16:19,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/41cde875067a49659089e0277e09ce0a is 50, key is test_row_0/C:col10/1731132979117/Put/seqid=0 2024-11-09T06:16:19,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T06:16:19,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:19,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:19,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:19,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742023_1199 (size=12301) 2024-11-09T06:16:19,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/41cde875067a49659089e0277e09ce0a 2024-11-09T06:16:19,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/ac495e2ee18148ce89f451305a3418a8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ac495e2ee18148ce89f451305a3418a8 2024-11-09T06:16:19,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ac495e2ee18148ce89f451305a3418a8, entries=250, sequenceid=334, filesize=47.5 K 2024-11-09T06:16:19,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/1a79b4e768c04e5992641ee30e6c465f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/1a79b4e768c04e5992641ee30e6c465f 2024-11-09T06:16:19,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/1a79b4e768c04e5992641ee30e6c465f, entries=150, sequenceid=334, filesize=12.0 K 2024-11-09T06:16:19,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/41cde875067a49659089e0277e09ce0a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/41cde875067a49659089e0277e09ce0a 2024-11-09T06:16:19,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/41cde875067a49659089e0277e09ce0a, entries=150, sequenceid=334, filesize=12.0 K 2024-11-09T06:16:19,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for f48863a85e472c28d23fb8419a1993ee in 617ms, sequenceid=334, compaction requested=false 2024-11-09T06:16:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:19,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:16:19,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:19,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:19,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:19,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:19,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:19,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:19,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T06:16:19,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:19,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:19,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:19,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133039816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133039817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133039820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133039821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T06:16:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133039823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109492c7d0f87a840cf883b91132727be76_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132979793/Put/seqid=0 2024-11-09T06:16:19,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742024_1200 (size=14994) 2024-11-09T06:16:19,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133039933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133039933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133039943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133039943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133039942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:19,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T06:16:19,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:19,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:20,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T06:16:20,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:20,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:20,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133040143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133040144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133040163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133040169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133040169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,244 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:20,249 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109492c7d0f87a840cf883b91132727be76_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109492c7d0f87a840cf883b91132727be76_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:20,252 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/c22f477cf9cb4c4689896e763dc77ad7, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:20,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/c22f477cf9cb4c4689896e763dc77ad7 is 175, key is test_row_0/A:col10/1731132979793/Put/seqid=0 2024-11-09T06:16:20,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742025_1201 (size=39949) 2024-11-09T06:16:20,262 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/c22f477cf9cb4c4689896e763dc77ad7 2024-11-09T06:16:20,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/7b533dabe59e4b609e4d9b7445d2aded is 50, key is test_row_0/B:col10/1731132979793/Put/seqid=0 2024-11-09T06:16:20,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742026_1202 (size=12301) 2024-11-09T06:16:20,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/7b533dabe59e4b609e4d9b7445d2aded 2024-11-09T06:16:20,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T06:16:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:20,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:20,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/0615d120fc8b43f4ab74a3411aee4973 is 50, key is test_row_0/C:col10/1731132979793/Put/seqid=0 2024-11-09T06:16:20,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742027_1203 (size=12301) 2024-11-09T06:16:20,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/0615d120fc8b43f4ab74a3411aee4973 2024-11-09T06:16:20,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/c22f477cf9cb4c4689896e763dc77ad7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c22f477cf9cb4c4689896e763dc77ad7 2024-11-09T06:16:20,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c22f477cf9cb4c4689896e763dc77ad7, entries=200, sequenceid=355, filesize=39.0 K 2024-11-09T06:16:20,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/7b533dabe59e4b609e4d9b7445d2aded as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/7b533dabe59e4b609e4d9b7445d2aded 2024-11-09T06:16:20,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/7b533dabe59e4b609e4d9b7445d2aded, entries=150, sequenceid=355, filesize=12.0 K 2024-11-09T06:16:20,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/0615d120fc8b43f4ab74a3411aee4973 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/0615d120fc8b43f4ab74a3411aee4973 2024-11-09T06:16:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T06:16:20,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/0615d120fc8b43f4ab74a3411aee4973, entries=150, sequenceid=355, filesize=12.0 K 2024-11-09T06:16:20,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f48863a85e472c28d23fb8419a1993ee in 556ms, sequenceid=355, compaction requested=true 2024-11-09T06:16:20,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:20,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:20,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:20,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:20,352 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:20,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:20,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:20,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:20,352 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:20,353 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120457 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:20,353 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:20,353 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/B is initiating minor compaction (all files) 2024-11-09T06:16:20,353 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/A is initiating minor compaction (all files) 2024-11-09T06:16:20,354 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/B in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:20,354 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/A in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:20,354 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/41b5ddfcbd3f4fc689a0686987a272a6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ac495e2ee18148ce89f451305a3418a8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c22f477cf9cb4c4689896e763dc77ad7] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=117.6 K 2024-11-09T06:16:20,354 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/87a276bc2b624d6a93455d706fe60d69, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/1a79b4e768c04e5992641ee30e6c465f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/7b533dabe59e4b609e4d9b7445d2aded] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=36.6 K 2024-11-09T06:16:20,354 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:20,354 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/41b5ddfcbd3f4fc689a0686987a272a6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ac495e2ee18148ce89f451305a3418a8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c22f477cf9cb4c4689896e763dc77ad7] 2024-11-09T06:16:20,354 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 87a276bc2b624d6a93455d706fe60d69, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731132977572 2024-11-09T06:16:20,355 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41b5ddfcbd3f4fc689a0686987a272a6, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731132977572 2024-11-09T06:16:20,355 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a79b4e768c04e5992641ee30e6c465f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731132978718 2024-11-09T06:16:20,355 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac495e2ee18148ce89f451305a3418a8, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731132978718 2024-11-09T06:16:20,356 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b533dabe59e4b609e4d9b7445d2aded, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731132979160 2024-11-09T06:16:20,356 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c22f477cf9cb4c4689896e763dc77ad7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731132979160 2024-11-09T06:16:20,369 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:20,370 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#B#compaction#167 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:20,370 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/a7329b2c98c94be287c8ec515a1b6567 is 50, key is test_row_0/B:col10/1731132979793/Put/seqid=0 2024-11-09T06:16:20,373 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109d3a15959d16f47508d90c7cdc19f598e_f48863a85e472c28d23fb8419a1993ee store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:20,375 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109d3a15959d16f47508d90c7cdc19f598e_f48863a85e472c28d23fb8419a1993ee, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:20,375 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d3a15959d16f47508d90c7cdc19f598e_f48863a85e472c28d23fb8419a1993ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:20,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742028_1204 (size=13017) 2024-11-09T06:16:20,423 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/a7329b2c98c94be287c8ec515a1b6567 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a7329b2c98c94be287c8ec515a1b6567 2024-11-09T06:16:20,430 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/B of f48863a85e472c28d23fb8419a1993ee into a7329b2c98c94be287c8ec515a1b6567(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:20,430 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:20,430 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/B, priority=13, startTime=1731132980352; duration=0sec 2024-11-09T06:16:20,431 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:20,431 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:B 2024-11-09T06:16:20,431 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:20,433 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:20,433 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/C is initiating minor compaction (all files) 2024-11-09T06:16:20,434 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/C in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:20,434 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/20b15da6eb204be7bd15ea39e7f12478, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/41cde875067a49659089e0277e09ce0a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/0615d120fc8b43f4ab74a3411aee4973] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=36.6 K 2024-11-09T06:16:20,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742029_1205 (size=4469) 2024-11-09T06:16:20,435 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b15da6eb204be7bd15ea39e7f12478, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731132977572 2024-11-09T06:16:20,436 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 41cde875067a49659089e0277e09ce0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731132978718 2024-11-09T06:16:20,436 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0615d120fc8b43f4ab74a3411aee4973, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731132979160 2024-11-09T06:16:20,437 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#A#compaction#168 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:20,438 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/2fab2881df994652a85fe1306a68e0a6 is 175, key is test_row_0/A:col10/1731132979793/Put/seqid=0 2024-11-09T06:16:20,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T06:16:20,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:20,452 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T06:16:20,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:20,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:20,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:20,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:20,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:20,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:20,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:20,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742030_1206 (size=31971) 2024-11-09T06:16:20,458 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#C#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:20,458 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/467bd02324b147e7bf5f7cc2e58e2eeb is 50, key is test_row_0/C:col10/1731132979793/Put/seqid=0 2024-11-09T06:16:20,477 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/2fab2881df994652a85fe1306a68e0a6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fab2881df994652a85fe1306a68e0a6 2024-11-09T06:16:20,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109053e385a385f4a7eb3b9d04d1fec8b52_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132979817/Put/seqid=0 2024-11-09T06:16:20,487 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/A of f48863a85e472c28d23fb8419a1993ee into 2fab2881df994652a85fe1306a68e0a6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:20,487 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:20,487 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/A, priority=13, startTime=1731132980352; duration=0sec 2024-11-09T06:16:20,487 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:20,487 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:A 2024-11-09T06:16:20,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742031_1207 (size=12983) 2024-11-09T06:16:20,499 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/467bd02324b147e7bf5f7cc2e58e2eeb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/467bd02324b147e7bf5f7cc2e58e2eeb 2024-11-09T06:16:20,510 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/C of f48863a85e472c28d23fb8419a1993ee into 467bd02324b147e7bf5f7cc2e58e2eeb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:20,510 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:20,510 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/C, priority=13, startTime=1731132980352; duration=0sec 2024-11-09T06:16:20,510 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:20,510 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:C 2024-11-09T06:16:20,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133040496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133040507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133040509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133040510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133040512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742032_1208 (size=12454) 2024-11-09T06:16:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:20,537 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109053e385a385f4a7eb3b9d04d1fec8b52_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109053e385a385f4a7eb3b9d04d1fec8b52_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:20,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/f3ee623523f64392b20eb72f8053a883, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:20,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/f3ee623523f64392b20eb72f8053a883 is 175, key is test_row_0/A:col10/1731132979817/Put/seqid=0 2024-11-09T06:16:20,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742033_1209 (size=31255) 2024-11-09T06:16:20,568 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/f3ee623523f64392b20eb72f8053a883 2024-11-09T06:16:20,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/fccf63a58f54461bae00579717ffab40 is 50, key is test_row_0/B:col10/1731132979817/Put/seqid=0 2024-11-09T06:16:20,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742034_1210 (size=12301) 2024-11-09T06:16:20,588 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/fccf63a58f54461bae00579717ffab40 2024-11-09T06:16:20,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/3eb77f38fb74491bb29eb12716a24be5 is 50, key is test_row_0/C:col10/1731132979817/Put/seqid=0 2024-11-09T06:16:20,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742035_1211 (size=12301) 2024-11-09T06:16:20,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133040623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133040623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133040623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133040623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133040623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133040833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133040834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133040834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133040834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133040834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,024 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/3eb77f38fb74491bb29eb12716a24be5 2024-11-09T06:16:21,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/f3ee623523f64392b20eb72f8053a883 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f3ee623523f64392b20eb72f8053a883 2024-11-09T06:16:21,046 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f3ee623523f64392b20eb72f8053a883, entries=150, sequenceid=368, filesize=30.5 K 2024-11-09T06:16:21,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/fccf63a58f54461bae00579717ffab40 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fccf63a58f54461bae00579717ffab40 2024-11-09T06:16:21,057 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fccf63a58f54461bae00579717ffab40, entries=150, sequenceid=368, filesize=12.0 K 2024-11-09T06:16:21,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/3eb77f38fb74491bb29eb12716a24be5 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3eb77f38fb74491bb29eb12716a24be5 2024-11-09T06:16:21,064 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3eb77f38fb74491bb29eb12716a24be5, entries=150, sequenceid=368, filesize=12.0 K 2024-11-09T06:16:21,066 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f48863a85e472c28d23fb8419a1993ee in 614ms, sequenceid=368, compaction requested=false 2024-11-09T06:16:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-09T06:16:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-09T06:16:21,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-09T06:16:21,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8710 sec 2024-11-09T06:16:21,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8810 sec 2024-11-09T06:16:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:21,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132980505, max timestamp=1731132981145Num uniques 859; 2024-11-09T06:16:21,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:21,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=96.16 KB heapSize=282.03 KB 2024-11-09T06:16:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:21,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d04cd99b629446189ee90070af259d5c_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132981149/Put/seqid=0 2024-11-09T06:16:21,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742036_1212 (size=12454) 2024-11-09T06:16:21,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133041172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133041175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133041178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133041181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133041185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133041292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133041292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133041292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133041293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133041303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T06:16:21,364 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:21,364 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-09T06:16:21,367 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:21,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T06:16:21,368 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:21,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T06:16:21,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133041503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133041503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133041503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133041503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133041513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T06:16:21,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:21,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:21,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:21,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:21,583 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d04cd99b629446189ee90070af259d5c_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d04cd99b629446189ee90070af259d5c_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:21,584 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/5ff97676114648f0a1eabc1f2f8b8752, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:21,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/5ff97676114648f0a1eabc1f2f8b8752 is 175, key is test_row_0/A:col10/1731132981149/Put/seqid=0 2024-11-09T06:16:21,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742037_1213 (size=31255) 2024-11-09T06:16:21,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T06:16:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:21,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T06:16:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:21,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133041813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133041813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133041813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133041822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133041823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:21,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T06:16:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:21,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:21,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:21,992 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/5ff97676114648f0a1eabc1f2f8b8752 2024-11-09T06:16:22,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T06:16:22,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:22,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T06:16:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/4f84d3ea9e0943a49894bd2e0a150d27 is 50, key is test_row_0/B:col10/1731132981149/Put/seqid=0 2024-11-09T06:16:22,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742038_1214 (size=12301) 2024-11-09T06:16:22,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T06:16:22,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:22,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T06:16:22,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:22,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133042323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:22,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133042328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:22,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133042328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:22,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133042332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:22,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133042333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:22,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/4f84d3ea9e0943a49894bd2e0a150d27 2024-11-09T06:16:22,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/22774dc052934dda8cd302d13762f820 is 50, key is test_row_0/C:col10/1731132981149/Put/seqid=0 2024-11-09T06:16:22,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742039_1215 (size=12301) 2024-11-09T06:16:22,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/22774dc052934dda8cd302d13762f820 2024-11-09T06:16:22,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/5ff97676114648f0a1eabc1f2f8b8752 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5ff97676114648f0a1eabc1f2f8b8752 2024-11-09T06:16:22,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5ff97676114648f0a1eabc1f2f8b8752, entries=150, sequenceid=394, filesize=30.5 K 2024-11-09T06:16:22,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T06:16:22,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:22,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/4f84d3ea9e0943a49894bd2e0a150d27 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/4f84d3ea9e0943a49894bd2e0a150d27 2024-11-09T06:16:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:22,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/4f84d3ea9e0943a49894bd2e0a150d27, entries=150, sequenceid=394, filesize=12.0 K 2024-11-09T06:16:22,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/22774dc052934dda8cd302d13762f820 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/22774dc052934dda8cd302d13762f820 2024-11-09T06:16:22,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/22774dc052934dda8cd302d13762f820, entries=150, sequenceid=394, filesize=12.0 K 2024-11-09T06:16:22,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~109.58 KB/112210, heapSize ~321.83 KB/329552, currentSize=67.09 KB/68700 for f48863a85e472c28d23fb8419a1993ee in 1351ms, sequenceid=394, compaction requested=true 2024-11-09T06:16:22,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:22,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:22,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:22,501 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:22,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:22,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:22,501 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:22,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:22,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:22,502 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:22,502 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/B is initiating minor compaction (all files) 2024-11-09T06:16:22,502 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/B in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,503 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a7329b2c98c94be287c8ec515a1b6567, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fccf63a58f54461bae00579717ffab40, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/4f84d3ea9e0943a49894bd2e0a150d27] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=36.7 K 2024-11-09T06:16:22,503 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a7329b2c98c94be287c8ec515a1b6567, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731132979160 2024-11-09T06:16:22,503 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting fccf63a58f54461bae00579717ffab40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731132979805 2024-11-09T06:16:22,504 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:22,504 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/A is initiating minor compaction (all files) 2024-11-09T06:16:22,504 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/A in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,504 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fab2881df994652a85fe1306a68e0a6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f3ee623523f64392b20eb72f8053a883, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5ff97676114648f0a1eabc1f2f8b8752] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=92.3 K 2024-11-09T06:16:22,504 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,504 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fab2881df994652a85fe1306a68e0a6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f3ee623523f64392b20eb72f8053a883, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5ff97676114648f0a1eabc1f2f8b8752] 2024-11-09T06:16:22,504 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fab2881df994652a85fe1306a68e0a6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731132979160 2024-11-09T06:16:22,504 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f84d3ea9e0943a49894bd2e0a150d27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731132981145 2024-11-09T06:16:22,505 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3ee623523f64392b20eb72f8053a883, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731132979805 2024-11-09T06:16:22,505 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ff97676114648f0a1eabc1f2f8b8752, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731132981145 2024-11-09T06:16:22,512 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:22,520 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:22,521 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/8ada300bc0be46c08f82a6c22b805f94 is 50, key is test_row_0/B:col10/1731132981149/Put/seqid=0 2024-11-09T06:16:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T06:16:22,525 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110983f06e0f21dd496d938e5512ad672c70_f48863a85e472c28d23fb8419a1993ee store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:22,527 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110983f06e0f21dd496d938e5512ad672c70_f48863a85e472c28d23fb8419a1993ee, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:22,527 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110983f06e0f21dd496d938e5512ad672c70_f48863a85e472c28d23fb8419a1993ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:22,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742041_1217 (size=4469) 2024-11-09T06:16:22,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742040_1216 (size=13119) 2024-11-09T06:16:22,581 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/8ada300bc0be46c08f82a6c22b805f94 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/8ada300bc0be46c08f82a6c22b805f94 2024-11-09T06:16:22,589 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/B of f48863a85e472c28d23fb8419a1993ee into 8ada300bc0be46c08f82a6c22b805f94(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:22,589 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:22,589 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/B, priority=13, startTime=1731132982501; duration=0sec 2024-11-09T06:16:22,589 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:22,589 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:B 2024-11-09T06:16:22,590 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:22,593 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:22,593 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/C is initiating minor compaction (all files) 2024-11-09T06:16:22,593 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/C in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,593 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/467bd02324b147e7bf5f7cc2e58e2eeb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3eb77f38fb74491bb29eb12716a24be5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/22774dc052934dda8cd302d13762f820] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=36.7 K 2024-11-09T06:16:22,595 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 467bd02324b147e7bf5f7cc2e58e2eeb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731132979160 2024-11-09T06:16:22,595 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb77f38fb74491bb29eb12716a24be5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731132979805 2024-11-09T06:16:22,596 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 22774dc052934dda8cd302d13762f820, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731132981145 2024-11-09T06:16:22,605 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#C#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:22,606 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/57ebf08b6d7142f183381fc355502992 is 50, key is test_row_0/C:col10/1731132981149/Put/seqid=0 2024-11-09T06:16:22,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742042_1218 (size=13085) 2024-11-09T06:16:22,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T06:16:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,636 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:16:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:22,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:22,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:22,637 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/57ebf08b6d7142f183381fc355502992 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/57ebf08b6d7142f183381fc355502992 2024-11-09T06:16:22,648 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/C of f48863a85e472c28d23fb8419a1993ee into 57ebf08b6d7142f183381fc355502992(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:22,648 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:22,648 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/C, priority=13, startTime=1731132982501; duration=0sec 2024-11-09T06:16:22,648 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:22,648 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:C 2024-11-09T06:16:22,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109614afc8afe9a4367af2ecfa65c50ba11_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132981176/Put/seqid=0 2024-11-09T06:16:22,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742043_1219 (size=12454) 2024-11-09T06:16:22,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,667 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109614afc8afe9a4367af2ecfa65c50ba11_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109614afc8afe9a4367af2ecfa65c50ba11_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:22,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/1dbbfbffd3d3434eb64bfb16caa00894, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:22,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/1dbbfbffd3d3434eb64bfb16caa00894 is 175, key is test_row_0/A:col10/1731132981176/Put/seqid=0 2024-11-09T06:16:22,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742044_1220 (size=31255) 2024-11-09T06:16:22,691 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/1dbbfbffd3d3434eb64bfb16caa00894 2024-11-09T06:16:22,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/0968cb3ca0e04c41afac55ebf788ec72 is 50, key is test_row_0/B:col10/1731132981176/Put/seqid=0 2024-11-09T06:16:22,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742045_1221 (size=12301) 2024-11-09T06:16:22,716 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/0968cb3ca0e04c41afac55ebf788ec72 2024-11-09T06:16:22,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/3dd4a5e2963a4646858d663a3eac0300 is 50, key is test_row_0/C:col10/1731132981176/Put/seqid=0 2024-11-09T06:16:22,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742046_1222 (size=12301) 2024-11-09T06:16:22,733 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/3dd4a5e2963a4646858d663a3eac0300 2024-11-09T06:16:22,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/1dbbfbffd3d3434eb64bfb16caa00894 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1dbbfbffd3d3434eb64bfb16caa00894 2024-11-09T06:16:22,750 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1dbbfbffd3d3434eb64bfb16caa00894, entries=150, sequenceid=408, filesize=30.5 K 2024-11-09T06:16:22,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/0968cb3ca0e04c41afac55ebf788ec72 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/0968cb3ca0e04c41afac55ebf788ec72 2024-11-09T06:16:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,758 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/0968cb3ca0e04c41afac55ebf788ec72, entries=150, sequenceid=408, filesize=12.0 K 2024-11-09T06:16:22,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/3dd4a5e2963a4646858d663a3eac0300 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3dd4a5e2963a4646858d663a3eac0300 2024-11-09T06:16:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,766 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3dd4a5e2963a4646858d663a3eac0300, entries=150, sequenceid=408, filesize=12.0 K 2024-11-09T06:16:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,767 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for f48863a85e472c28d23fb8419a1993ee in 131ms, sequenceid=408, compaction requested=false 2024-11-09T06:16:22,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:22,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-09T06:16:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-09T06:16:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-09T06:16:22,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-11-09T06:16:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4090 sec 2024-11-09T06:16:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,956 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#A#compaction#176 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,957 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/f989459e4d524e1ea0114110d6491056 is 175, key is test_row_0/A:col10/1731132981149/Put/seqid=0 2024-11-09T06:16:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742047_1223 (size=32073) 2024-11-09T06:16:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,378 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/f989459e4d524e1ea0114110d6491056 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f989459e4d524e1ea0114110d6491056 2024-11-09T06:16:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,395 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/A of f48863a85e472c28d23fb8419a1993ee into f989459e4d524e1ea0114110d6491056(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:23,395 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:23,395 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/A, priority=13, startTime=1731132982501; duration=0sec 2024-11-09T06:16:23,395 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:23,395 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:A 2024-11-09T06:16:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:16:23,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:23,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:23,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:23,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:23,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:23,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:23,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cec84d1afe1f4b91a3d39c0283678c5b_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132983461/Put/seqid=0 2024-11-09T06:16:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T06:16:23,534 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:23,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-09T06:16:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T06:16:23,541 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:23,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742048_1224 (size=20074) 2024-11-09T06:16:23,543 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:23,543 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:23,543 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:23,552 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cec84d1afe1f4b91a3d39c0283678c5b_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cec84d1afe1f4b91a3d39c0283678c5b_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:23,554 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/1e409faf5ed84aacaa3855ff1efdcfc6, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:23,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/1e409faf5ed84aacaa3855ff1efdcfc6 is 175, key is test_row_0/A:col10/1731132983461/Put/seqid=0 2024-11-09T06:16:23,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731132983533, max timestamp=1731132983559Num uniques 859; 2024-11-09T06:16:23,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133043540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731132983533, max timestamp=1731132983559Num uniques 859; 2024-11-09T06:16:23,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:23,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731132983533, max timestamp=1731132983559Num uniques 859; 2024-11-09T06:16:23,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:23,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742049_1225 (size=57333) 2024-11-09T06:16:23,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=420, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/1e409faf5ed84aacaa3855ff1efdcfc6 2024-11-09T06:16:23,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/57ca3b66af0549109db09ea407124fce is 50, key is test_row_0/B:col10/1731132983461/Put/seqid=0 2024-11-09T06:16:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T06:16:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133043646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742050_1226 (size=12301) 2024-11-09T06:16:23,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133043651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133043654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133043654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133043674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T06:16:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:23,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:23,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133043763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133043783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133043784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133043785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T06:16:23,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T06:16:23,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:23,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:23,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:23,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:23,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:23,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133043904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133043983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133043994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133043994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:24,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133044002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:24,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T06:16:24,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:24,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/57ca3b66af0549109db09ea407124fce 2024-11-09T06:16:24,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/511924b1f3fb4cb88842bb7bcbd8ed02 is 50, key is test_row_0/C:col10/1731132983461/Put/seqid=0 2024-11-09T06:16:24,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742051_1227 (size=12301) 2024-11-09T06:16:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T06:16:24,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T06:16:24,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53358 deadline: 1731133044213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:24,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:24,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53370 deadline: 1731133044292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:24,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53396 deadline: 1731133044303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53426 deadline: 1731133044303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:24,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53382 deadline: 1731133044312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:24,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:24,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:16:24,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T06:16:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,373 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-09T06:16:24,373 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:24,373 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,373 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:24,376 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-09T06:16:24,376 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:24,376 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,377 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,377 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-09T06:16:24,377 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:24,377 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,377 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:24,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:24,378 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-09T06:16:24,378 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:24,378 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,378 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:24,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T06:16:24,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:24,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:24,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/511924b1f3fb4cb88842bb7bcbd8ed02 2024-11-09T06:16:24,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/1e409faf5ed84aacaa3855ff1efdcfc6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1e409faf5ed84aacaa3855ff1efdcfc6 2024-11-09T06:16:24,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1e409faf5ed84aacaa3855ff1efdcfc6, entries=300, sequenceid=420, filesize=56.0 K 2024-11-09T06:16:24,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/57ca3b66af0549109db09ea407124fce as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/57ca3b66af0549109db09ea407124fce 2024-11-09T06:16:24,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/57ca3b66af0549109db09ea407124fce, entries=150, sequenceid=420, filesize=12.0 K 2024-11-09T06:16:24,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/511924b1f3fb4cb88842bb7bcbd8ed02 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/511924b1f3fb4cb88842bb7bcbd8ed02 2024-11-09T06:16:24,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/511924b1f3fb4cb88842bb7bcbd8ed02, entries=150, sequenceid=420, filesize=12.0 K 2024-11-09T06:16:24,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f48863a85e472c28d23fb8419a1993ee in 1044ms, sequenceid=420, compaction requested=true 2024-11-09T06:16:24,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:24,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:24,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:24,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:24,509 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:24,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:24,509 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:24,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f48863a85e472c28d23fb8419a1993ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:24,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:24,510 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:24,510 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:24,510 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/A is initiating minor compaction (all files) 2024-11-09T06:16:24,510 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/B is initiating minor compaction (all files) 2024-11-09T06:16:24,510 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/B in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,510 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/A in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,510 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/8ada300bc0be46c08f82a6c22b805f94, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/0968cb3ca0e04c41afac55ebf788ec72, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/57ca3b66af0549109db09ea407124fce] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=36.8 K 2024-11-09T06:16:24,510 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f989459e4d524e1ea0114110d6491056, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1dbbfbffd3d3434eb64bfb16caa00894, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1e409faf5ed84aacaa3855ff1efdcfc6] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=117.8 K 2024-11-09T06:16:24,510 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,510 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f989459e4d524e1ea0114110d6491056, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1dbbfbffd3d3434eb64bfb16caa00894, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1e409faf5ed84aacaa3855ff1efdcfc6] 2024-11-09T06:16:24,510 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ada300bc0be46c08f82a6c22b805f94, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731132981145 2024-11-09T06:16:24,510 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f989459e4d524e1ea0114110d6491056, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731132981145 2024-11-09T06:16:24,511 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0968cb3ca0e04c41afac55ebf788ec72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731132981176 2024-11-09T06:16:24,511 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dbbfbffd3d3434eb64bfb16caa00894, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731132981176 2024-11-09T06:16:24,511 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ca3b66af0549109db09ea407124fce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731132983441 2024-11-09T06:16:24,511 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e409faf5ed84aacaa3855ff1efdcfc6, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731132983431 2024-11-09T06:16:24,522 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:24,523 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#B#compaction#185 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:24,524 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/b498817331d049e29d81023d5b241da4 is 50, key is test_row_0/B:col10/1731132983461/Put/seqid=0 2024-11-09T06:16:24,525 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411092a573202ca3c4353b267e854b748f914_f48863a85e472c28d23fb8419a1993ee store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:24,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742052_1228 (size=13221) 2024-11-09T06:16:24,540 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/b498817331d049e29d81023d5b241da4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/b498817331d049e29d81023d5b241da4 2024-11-09T06:16:24,550 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/B of f48863a85e472c28d23fb8419a1993ee into b498817331d049e29d81023d5b241da4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:24,550 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:24,550 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/B, priority=13, startTime=1731132984509; duration=0sec 2024-11-09T06:16:24,550 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:24,550 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:B 2024-11-09T06:16:24,551 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:24,552 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:24,552 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f48863a85e472c28d23fb8419a1993ee/C is initiating minor compaction (all files) 2024-11-09T06:16:24,552 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f48863a85e472c28d23fb8419a1993ee/C in TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,553 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/57ebf08b6d7142f183381fc355502992, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3dd4a5e2963a4646858d663a3eac0300, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/511924b1f3fb4cb88842bb7bcbd8ed02] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp, totalSize=36.8 K 2024-11-09T06:16:24,554 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ebf08b6d7142f183381fc355502992, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731132981145 2024-11-09T06:16:24,554 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dd4a5e2963a4646858d663a3eac0300, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731132981176 2024-11-09T06:16:24,555 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 511924b1f3fb4cb88842bb7bcbd8ed02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731132983441 2024-11-09T06:16:24,572 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#C#compaction#187 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:24,573 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/d33809fd12c247a2b81c437871fbcfc7 is 50, key is test_row_0/C:col10/1731132983461/Put/seqid=0 2024-11-09T06:16:24,573 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411092a573202ca3c4353b267e854b748f914_f48863a85e472c28d23fb8419a1993ee, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:24,573 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092a573202ca3c4353b267e854b748f914_f48863a85e472c28d23fb8419a1993ee because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:24,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742053_1229 (size=13187) 2024-11-09T06:16:24,590 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/d33809fd12c247a2b81c437871fbcfc7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/d33809fd12c247a2b81c437871fbcfc7 2024-11-09T06:16:24,596 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/C of f48863a85e472c28d23fb8419a1993ee into d33809fd12c247a2b81c437871fbcfc7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:24,596 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:24,596 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/C, priority=13, startTime=1731132984509; duration=0sec 2024-11-09T06:16:24,596 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:24,596 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:C 2024-11-09T06:16:24,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742054_1230 (size=4469) 2024-11-09T06:16:24,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T06:16:24,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:24,636 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-09T06:16:24,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:24,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:24,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:24,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109189b665d27034e689bda6baf0f1bd4b4_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132983635/Put/seqid=0 2024-11-09T06:16:24,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742055_1231 (size=12454) 2024-11-09T06:16:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T06:16:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:24,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. as already flushing 2024-11-09T06:16:24,734 INFO [Thread-619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-619. 2024-11-09T06:16:24,734 DEBUG [Thread-619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:24,734 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,734 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,734 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:24,807 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-11-09T06:16:24,807 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:24,807 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,808 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:24,816 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-11-09T06:16:24,816 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:24,816 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,816 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:24,826 INFO [Thread-617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-617. 2024-11-09T06:16:24,826 DEBUG [Thread-617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:24,826 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,826 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-11-09T06:16:24,826 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:24,826 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,826 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,826 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:24,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:24,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:25,006 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f48863a85e472c28d23fb8419a1993ee#A#compaction#186 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:25,006 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/eb67e6e7ecf84fce80842c0dc8bf0faf is 175, key is test_row_0/A:col10/1731132983461/Put/seqid=0 2024-11-09T06:16:25,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742056_1232 (size=32175) 2024-11-09T06:16:25,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:25,057 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109189b665d27034e689bda6baf0f1bd4b4_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109189b665d27034e689bda6baf0f1bd4b4_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:25,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/de1db91542b64987af4aa562716eee1b, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:25,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/de1db91542b64987af4aa562716eee1b is 175, key is test_row_0/A:col10/1731132983635/Put/seqid=0 2024-11-09T06:16:25,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742057_1233 (size=31255) 2024-11-09T06:16:25,066 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=458, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/de1db91542b64987af4aa562716eee1b 2024-11-09T06:16:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/506fb9f8f58b4b819a1a4b7800ed9d29 is 50, key is test_row_0/B:col10/1731132983635/Put/seqid=0 2024-11-09T06:16:25,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742058_1234 (size=12301) 2024-11-09T06:16:25,417 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/eb67e6e7ecf84fce80842c0dc8bf0faf as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/eb67e6e7ecf84fce80842c0dc8bf0faf 2024-11-09T06:16:25,423 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f48863a85e472c28d23fb8419a1993ee/A of f48863a85e472c28d23fb8419a1993ee into eb67e6e7ecf84fce80842c0dc8bf0faf(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:25,423 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:25,423 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., storeName=f48863a85e472c28d23fb8419a1993ee/A, priority=13, startTime=1731132984509; duration=0sec 2024-11-09T06:16:25,423 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:25,423 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f48863a85e472c28d23fb8419a1993ee:A 2024-11-09T06:16:25,482 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/506fb9f8f58b4b819a1a4b7800ed9d29 2024-11-09T06:16:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/44cd14cae4264d069451a4e90206ad38 is 50, key is test_row_0/C:col10/1731132983635/Put/seqid=0 2024-11-09T06:16:25,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742059_1235 (size=12301) 2024-11-09T06:16:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T06:16:25,901 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/44cd14cae4264d069451a4e90206ad38 2024-11-09T06:16:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/de1db91542b64987af4aa562716eee1b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/de1db91542b64987af4aa562716eee1b 2024-11-09T06:16:25,915 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/de1db91542b64987af4aa562716eee1b, entries=150, sequenceid=458, filesize=30.5 K 2024-11-09T06:16:25,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/506fb9f8f58b4b819a1a4b7800ed9d29 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/506fb9f8f58b4b819a1a4b7800ed9d29 2024-11-09T06:16:25,920 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/506fb9f8f58b4b819a1a4b7800ed9d29, entries=150, sequenceid=458, filesize=12.0 K 2024-11-09T06:16:25,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/44cd14cae4264d069451a4e90206ad38 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/44cd14cae4264d069451a4e90206ad38 2024-11-09T06:16:25,924 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/44cd14cae4264d069451a4e90206ad38, entries=150, sequenceid=458, filesize=12.0 K 2024-11-09T06:16:25,925 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=33.54 KB/34350 for f48863a85e472c28d23fb8419a1993ee in 1289ms, sequenceid=458, compaction requested=false 2024-11-09T06:16:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f48863a85e472c28d23fb8419a1993ee: 2024-11-09T06:16:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-09T06:16:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-09T06:16:25,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-09T06:16:25,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3840 sec 2024-11-09T06:16:25,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.3930 sec 2024-11-09T06:16:27,299 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T06:16:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T06:16:27,703 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1908 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1846 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 869 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2607 rows 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 872 2024-11-09T06:16:27,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2616 rows 2024-11-09T06:16:27,703 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T06:16:27,703 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T06:16:27,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:27,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:27,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:27,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T06:16:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T06:16:27,707 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731132987707"}]},"ts":"1731132987707"} 2024-11-09T06:16:27,709 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T06:16:27,709 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T06:16:27,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T06:16:27,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, UNASSIGN}] 2024-11-09T06:16:27,712 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, UNASSIGN 2024-11-09T06:16:27,712 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=f48863a85e472c28d23fb8419a1993ee, regionState=CLOSING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:27,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, UNASSIGN because future has completed 2024-11-09T06:16:27,714 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T06:16:27,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:16:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T06:16:27,866 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:27,866 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T06:16:27,866 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing f48863a85e472c28d23fb8419a1993ee, disabling compactions & flushes 2024-11-09T06:16:27,866 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:27,866 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:27,867 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. after waiting 0 ms 2024-11-09T06:16:27,867 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:27,867 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing f48863a85e472c28d23fb8419a1993ee 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T06:16:27,867 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=A 2024-11-09T06:16:27,867 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:27,867 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=B 2024-11-09T06:16:27,867 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:27,867 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f48863a85e472c28d23fb8419a1993ee, store=C 2024-11-09T06:16:27,867 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:27,880 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e2a0058b3cd8418d92abcdcd97a1f4a3_f48863a85e472c28d23fb8419a1993ee is 50, key is test_row_0/A:col10/1731132984823/Put/seqid=0 2024-11-09T06:16:27,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742060_1236 (size=12454) 2024-11-09T06:16:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T06:16:28,286 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:28,290 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e2a0058b3cd8418d92abcdcd97a1f4a3_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e2a0058b3cd8418d92abcdcd97a1f4a3_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:28,291 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/de7479854ba64a68b98f4055c01bca31, store: [table=TestAcidGuarantees family=A region=f48863a85e472c28d23fb8419a1993ee] 2024-11-09T06:16:28,292 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/de7479854ba64a68b98f4055c01bca31 is 175, key is test_row_0/A:col10/1731132984823/Put/seqid=0 2024-11-09T06:16:28,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742061_1237 (size=31255) 2024-11-09T06:16:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T06:16:28,697 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=467, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/de7479854ba64a68b98f4055c01bca31 2024-11-09T06:16:28,707 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/115abf0aa3b24bd28f653bf6add17929 is 50, key is test_row_0/B:col10/1731132984823/Put/seqid=0 2024-11-09T06:16:28,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742062_1238 (size=12301) 2024-11-09T06:16:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T06:16:29,112 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/115abf0aa3b24bd28f653bf6add17929 2024-11-09T06:16:29,120 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/de7480a2fb4247c1b51f66e609799f8c is 50, key is test_row_0/C:col10/1731132984823/Put/seqid=0 2024-11-09T06:16:29,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742063_1239 (size=12301) 2024-11-09T06:16:29,524 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/de7480a2fb4247c1b51f66e609799f8c 2024-11-09T06:16:29,528 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/A/de7479854ba64a68b98f4055c01bca31 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/de7479854ba64a68b98f4055c01bca31 2024-11-09T06:16:29,532 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/de7479854ba64a68b98f4055c01bca31, entries=150, sequenceid=467, filesize=30.5 K 2024-11-09T06:16:29,533 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/B/115abf0aa3b24bd28f653bf6add17929 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/115abf0aa3b24bd28f653bf6add17929 2024-11-09T06:16:29,537 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/115abf0aa3b24bd28f653bf6add17929, entries=150, sequenceid=467, filesize=12.0 K 2024-11-09T06:16:29,538 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/.tmp/C/de7480a2fb4247c1b51f66e609799f8c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/de7480a2fb4247c1b51f66e609799f8c 2024-11-09T06:16:29,542 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/de7480a2fb4247c1b51f66e609799f8c, entries=150, sequenceid=467, filesize=12.0 K 2024-11-09T06:16:29,543 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f48863a85e472c28d23fb8419a1993ee in 1675ms, sequenceid=467, compaction requested=true 2024-11-09T06:16:29,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ea2879829ce94a18b39755f24e997407, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fbea6b644ae46b2a6f8a20bab634857, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/0f57966d4ff74f46b37c861c5e8c5096, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5d61eca7284c42959015cc4d54da0563, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48c1f54e0adc42d9aaf09af240989d2c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/b179bc44e54443f49b22861632d3bff9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/978f41cd4f8f47e8bc28beb2536cf16b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48e7f92988484ee6bb6a5d0955742db9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f8dc3aeffb9545a3bf122452ef838532, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/a84a7eedf5ab406faaa564ab6ec0e0f7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/8d741e2e4265414296b801f02c3d69da, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/3b9efa401c3e4a97be4f292222b8d152, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/cf26e2350ccf4818bd54529c95b2f6dd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c4905c3443b243b6a39336501f70ede9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/354585acbaff4e4e8647539cfa320de6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/6b08ff941adb494db7a1784a17d1d64f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/4a7c0d8f42874609ae1496abc709d64a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/41b5ddfcbd3f4fc689a0686987a272a6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/9d719e9c2cdb4094a50e936dedd015bb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ac495e2ee18148ce89f451305a3418a8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c22f477cf9cb4c4689896e763dc77ad7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fab2881df994652a85fe1306a68e0a6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f3ee623523f64392b20eb72f8053a883, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f989459e4d524e1ea0114110d6491056, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5ff97676114648f0a1eabc1f2f8b8752, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1dbbfbffd3d3434eb64bfb16caa00894, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1e409faf5ed84aacaa3855ff1efdcfc6] to archive 2024-11-09T06:16:29,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:16:29,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ea2879829ce94a18b39755f24e997407 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ea2879829ce94a18b39755f24e997407 2024-11-09T06:16:29,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fbea6b644ae46b2a6f8a20bab634857 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fbea6b644ae46b2a6f8a20bab634857 2024-11-09T06:16:29,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/0f57966d4ff74f46b37c861c5e8c5096 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/0f57966d4ff74f46b37c861c5e8c5096 2024-11-09T06:16:29,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5d61eca7284c42959015cc4d54da0563 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5d61eca7284c42959015cc4d54da0563 2024-11-09T06:16:29,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48c1f54e0adc42d9aaf09af240989d2c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48c1f54e0adc42d9aaf09af240989d2c 2024-11-09T06:16:29,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/b179bc44e54443f49b22861632d3bff9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/b179bc44e54443f49b22861632d3bff9 2024-11-09T06:16:29,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/978f41cd4f8f47e8bc28beb2536cf16b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/978f41cd4f8f47e8bc28beb2536cf16b 2024-11-09T06:16:29,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48e7f92988484ee6bb6a5d0955742db9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/48e7f92988484ee6bb6a5d0955742db9 2024-11-09T06:16:29,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f8dc3aeffb9545a3bf122452ef838532 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f8dc3aeffb9545a3bf122452ef838532 2024-11-09T06:16:29,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/a84a7eedf5ab406faaa564ab6ec0e0f7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/a84a7eedf5ab406faaa564ab6ec0e0f7 2024-11-09T06:16:29,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/8d741e2e4265414296b801f02c3d69da to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/8d741e2e4265414296b801f02c3d69da 2024-11-09T06:16:29,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/3b9efa401c3e4a97be4f292222b8d152 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/3b9efa401c3e4a97be4f292222b8d152 2024-11-09T06:16:29,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/cf26e2350ccf4818bd54529c95b2f6dd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/cf26e2350ccf4818bd54529c95b2f6dd 2024-11-09T06:16:29,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c4905c3443b243b6a39336501f70ede9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c4905c3443b243b6a39336501f70ede9 2024-11-09T06:16:29,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/354585acbaff4e4e8647539cfa320de6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/354585acbaff4e4e8647539cfa320de6 2024-11-09T06:16:29,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/6b08ff941adb494db7a1784a17d1d64f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/6b08ff941adb494db7a1784a17d1d64f 2024-11-09T06:16:29,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/4a7c0d8f42874609ae1496abc709d64a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/4a7c0d8f42874609ae1496abc709d64a 2024-11-09T06:16:29,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/41b5ddfcbd3f4fc689a0686987a272a6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/41b5ddfcbd3f4fc689a0686987a272a6 2024-11-09T06:16:29,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/9d719e9c2cdb4094a50e936dedd015bb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/9d719e9c2cdb4094a50e936dedd015bb 2024-11-09T06:16:29,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ac495e2ee18148ce89f451305a3418a8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/ac495e2ee18148ce89f451305a3418a8 2024-11-09T06:16:29,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c22f477cf9cb4c4689896e763dc77ad7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/c22f477cf9cb4c4689896e763dc77ad7 2024-11-09T06:16:29,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fab2881df994652a85fe1306a68e0a6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/2fab2881df994652a85fe1306a68e0a6 2024-11-09T06:16:29,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f3ee623523f64392b20eb72f8053a883 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f3ee623523f64392b20eb72f8053a883 2024-11-09T06:16:29,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f989459e4d524e1ea0114110d6491056 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/f989459e4d524e1ea0114110d6491056 2024-11-09T06:16:29,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5ff97676114648f0a1eabc1f2f8b8752 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/5ff97676114648f0a1eabc1f2f8b8752 2024-11-09T06:16:29,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1dbbfbffd3d3434eb64bfb16caa00894 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1dbbfbffd3d3434eb64bfb16caa00894 2024-11-09T06:16:29,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1e409faf5ed84aacaa3855ff1efdcfc6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/1e409faf5ed84aacaa3855ff1efdcfc6 2024-11-09T06:16:29,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/ef771a86c19544fd8a5194a4d0a7a0ff, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6cdac06a2c344419a71fde6286ff7178, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/5ec3948c3300409eb09dec9f611ae093, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/2f9005a2f62e49d7ac8e885791c46139, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6463b345fc034a76b4312c8d2b6e8842, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/af8cdc71e7d5423eb88b0ff2023cd26c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/28ac651612284316b2bcd149745884bf, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a108e19c5d33404091c2b75bb234138e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a81786aaea954958af235a628e08fa39, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/b09d4e92b44c4c3482ee3dc5209c8f15, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fa6f38078eca4554840da66186de0e01, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/16ebe8d02ae0454fa6c618e81427e32d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/32fcf8fb928646a99f643cc058d88308, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/def9943d9b8443d18860780d14bffb27, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/8a9a22ce4d5249d29ba8eab4c6e58694, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/3d7068c90f3248278300c63495accb61, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/76cff67c97cc4986bfc5599c0380d19f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/87a276bc2b624d6a93455d706fe60d69, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/5ef83a2fb59a48b18fd88944d32c6ed7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/1a79b4e768c04e5992641ee30e6c465f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a7329b2c98c94be287c8ec515a1b6567, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/7b533dabe59e4b609e4d9b7445d2aded, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fccf63a58f54461bae00579717ffab40, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/8ada300bc0be46c08f82a6c22b805f94, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/4f84d3ea9e0943a49894bd2e0a150d27, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/0968cb3ca0e04c41afac55ebf788ec72, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/57ca3b66af0549109db09ea407124fce] to archive 2024-11-09T06:16:29,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:16:29,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/ef771a86c19544fd8a5194a4d0a7a0ff to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/ef771a86c19544fd8a5194a4d0a7a0ff 2024-11-09T06:16:29,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6cdac06a2c344419a71fde6286ff7178 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6cdac06a2c344419a71fde6286ff7178 2024-11-09T06:16:29,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/5ec3948c3300409eb09dec9f611ae093 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/5ec3948c3300409eb09dec9f611ae093 2024-11-09T06:16:29,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/2f9005a2f62e49d7ac8e885791c46139 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/2f9005a2f62e49d7ac8e885791c46139 2024-11-09T06:16:29,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6463b345fc034a76b4312c8d2b6e8842 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/6463b345fc034a76b4312c8d2b6e8842 2024-11-09T06:16:29,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/af8cdc71e7d5423eb88b0ff2023cd26c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/af8cdc71e7d5423eb88b0ff2023cd26c 2024-11-09T06:16:29,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/28ac651612284316b2bcd149745884bf to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/28ac651612284316b2bcd149745884bf 2024-11-09T06:16:29,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a108e19c5d33404091c2b75bb234138e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a108e19c5d33404091c2b75bb234138e 2024-11-09T06:16:29,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a81786aaea954958af235a628e08fa39 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a81786aaea954958af235a628e08fa39 2024-11-09T06:16:29,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/b09d4e92b44c4c3482ee3dc5209c8f15 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/b09d4e92b44c4c3482ee3dc5209c8f15 2024-11-09T06:16:29,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fa6f38078eca4554840da66186de0e01 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fa6f38078eca4554840da66186de0e01 2024-11-09T06:16:29,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/16ebe8d02ae0454fa6c618e81427e32d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/16ebe8d02ae0454fa6c618e81427e32d 2024-11-09T06:16:29,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/32fcf8fb928646a99f643cc058d88308 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/32fcf8fb928646a99f643cc058d88308 2024-11-09T06:16:29,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/def9943d9b8443d18860780d14bffb27 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/def9943d9b8443d18860780d14bffb27 2024-11-09T06:16:29,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/8a9a22ce4d5249d29ba8eab4c6e58694 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/8a9a22ce4d5249d29ba8eab4c6e58694 2024-11-09T06:16:29,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/3d7068c90f3248278300c63495accb61 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/3d7068c90f3248278300c63495accb61 2024-11-09T06:16:29,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/76cff67c97cc4986bfc5599c0380d19f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/76cff67c97cc4986bfc5599c0380d19f 2024-11-09T06:16:29,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/87a276bc2b624d6a93455d706fe60d69 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/87a276bc2b624d6a93455d706fe60d69 2024-11-09T06:16:29,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/5ef83a2fb59a48b18fd88944d32c6ed7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/5ef83a2fb59a48b18fd88944d32c6ed7 2024-11-09T06:16:29,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/1a79b4e768c04e5992641ee30e6c465f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/1a79b4e768c04e5992641ee30e6c465f 2024-11-09T06:16:29,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a7329b2c98c94be287c8ec515a1b6567 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/a7329b2c98c94be287c8ec515a1b6567 2024-11-09T06:16:29,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/7b533dabe59e4b609e4d9b7445d2aded to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/7b533dabe59e4b609e4d9b7445d2aded 2024-11-09T06:16:29,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fccf63a58f54461bae00579717ffab40 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/fccf63a58f54461bae00579717ffab40 2024-11-09T06:16:29,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/8ada300bc0be46c08f82a6c22b805f94 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/8ada300bc0be46c08f82a6c22b805f94 2024-11-09T06:16:29,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/4f84d3ea9e0943a49894bd2e0a150d27 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/4f84d3ea9e0943a49894bd2e0a150d27 2024-11-09T06:16:29,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/0968cb3ca0e04c41afac55ebf788ec72 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/0968cb3ca0e04c41afac55ebf788ec72 2024-11-09T06:16:29,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/57ca3b66af0549109db09ea407124fce to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/57ca3b66af0549109db09ea407124fce 2024-11-09T06:16:29,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8569f62916f948b5b634bcb555a6c3a3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/a26f12e3354543d6a27d246a95ba65ab, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/94b2fdfc84b548eda306eb99fd611a30, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/1b357a37af844b68af29b85261bcb5f9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dc349f63c2c349f281e37d833cb1c770, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dcc6ae97728a4743be5090f5ab9925de, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/7153a6a1539746009e8250018772f713, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/326da85f7fa14260bb414a7eb84f1d33, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/d30d9a4108024a20bdb319563c8c3870, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/71ba62b4310d4877a564736a8d37e652, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/e2d64f49327d4a65b7f64ac6143adbb3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bd7c3c861e14c11b1ce27961ede3ee5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8b6e02be9fcc41f9ba27eee9f9c7c319, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/230f77b9c9ac4e1084327f09dc98c89c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bdd81a40e894c3587317045946aeaf1, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/9afd130693b648aa83fbc58f62fdaa76, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/20b15da6eb204be7bd15ea39e7f12478, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/b61a368943654ce59a232dfda1db034f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/41cde875067a49659089e0277e09ce0a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/467bd02324b147e7bf5f7cc2e58e2eeb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/0615d120fc8b43f4ab74a3411aee4973, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3eb77f38fb74491bb29eb12716a24be5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/57ebf08b6d7142f183381fc355502992, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/22774dc052934dda8cd302d13762f820, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3dd4a5e2963a4646858d663a3eac0300, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/511924b1f3fb4cb88842bb7bcbd8ed02] to archive 2024-11-09T06:16:29,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:16:29,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8569f62916f948b5b634bcb555a6c3a3 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8569f62916f948b5b634bcb555a6c3a3 2024-11-09T06:16:29,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/a26f12e3354543d6a27d246a95ba65ab to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/a26f12e3354543d6a27d246a95ba65ab 2024-11-09T06:16:29,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/94b2fdfc84b548eda306eb99fd611a30 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/94b2fdfc84b548eda306eb99fd611a30 2024-11-09T06:16:29,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/1b357a37af844b68af29b85261bcb5f9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/1b357a37af844b68af29b85261bcb5f9 2024-11-09T06:16:29,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dc349f63c2c349f281e37d833cb1c770 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dc349f63c2c349f281e37d833cb1c770 2024-11-09T06:16:29,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dcc6ae97728a4743be5090f5ab9925de to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/dcc6ae97728a4743be5090f5ab9925de 2024-11-09T06:16:29,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/7153a6a1539746009e8250018772f713 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/7153a6a1539746009e8250018772f713 2024-11-09T06:16:29,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/326da85f7fa14260bb414a7eb84f1d33 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/326da85f7fa14260bb414a7eb84f1d33 2024-11-09T06:16:29,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/d30d9a4108024a20bdb319563c8c3870 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/d30d9a4108024a20bdb319563c8c3870 2024-11-09T06:16:29,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/71ba62b4310d4877a564736a8d37e652 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/71ba62b4310d4877a564736a8d37e652 2024-11-09T06:16:29,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/e2d64f49327d4a65b7f64ac6143adbb3 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/e2d64f49327d4a65b7f64ac6143adbb3 2024-11-09T06:16:29,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bd7c3c861e14c11b1ce27961ede3ee5 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bd7c3c861e14c11b1ce27961ede3ee5 2024-11-09T06:16:29,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8b6e02be9fcc41f9ba27eee9f9c7c319 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/8b6e02be9fcc41f9ba27eee9f9c7c319 2024-11-09T06:16:29,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/230f77b9c9ac4e1084327f09dc98c89c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/230f77b9c9ac4e1084327f09dc98c89c 2024-11-09T06:16:29,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bdd81a40e894c3587317045946aeaf1 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/6bdd81a40e894c3587317045946aeaf1 2024-11-09T06:16:29,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/9afd130693b648aa83fbc58f62fdaa76 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/9afd130693b648aa83fbc58f62fdaa76 2024-11-09T06:16:29,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/20b15da6eb204be7bd15ea39e7f12478 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/20b15da6eb204be7bd15ea39e7f12478 2024-11-09T06:16:29,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/b61a368943654ce59a232dfda1db034f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/b61a368943654ce59a232dfda1db034f 2024-11-09T06:16:29,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/41cde875067a49659089e0277e09ce0a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/41cde875067a49659089e0277e09ce0a 2024-11-09T06:16:29,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/467bd02324b147e7bf5f7cc2e58e2eeb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/467bd02324b147e7bf5f7cc2e58e2eeb 2024-11-09T06:16:29,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/0615d120fc8b43f4ab74a3411aee4973 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/0615d120fc8b43f4ab74a3411aee4973 2024-11-09T06:16:29,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3eb77f38fb74491bb29eb12716a24be5 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3eb77f38fb74491bb29eb12716a24be5 2024-11-09T06:16:29,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/57ebf08b6d7142f183381fc355502992 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/57ebf08b6d7142f183381fc355502992 2024-11-09T06:16:29,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/22774dc052934dda8cd302d13762f820 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/22774dc052934dda8cd302d13762f820 2024-11-09T06:16:29,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3dd4a5e2963a4646858d663a3eac0300 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/3dd4a5e2963a4646858d663a3eac0300 2024-11-09T06:16:29,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/511924b1f3fb4cb88842bb7bcbd8ed02 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/511924b1f3fb4cb88842bb7bcbd8ed02 2024-11-09T06:16:29,645 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/recovered.edits/470.seqid, newMaxSeqId=470, maxSeqId=4 2024-11-09T06:16:29,646 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. 2024-11-09T06:16:29,646 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for f48863a85e472c28d23fb8419a1993ee: Waiting for close lock at 1731132987866Running coprocessor pre-close hooks at 1731132987866Disabling compacts and flushes for region at 1731132987866Disabling writes for close at 1731132987867 (+1 ms)Obtaining lock to block concurrent updates at 1731132987867Preparing flush snapshotting stores in f48863a85e472c28d23fb8419a1993ee at 1731132987867Finished memstore snapshotting TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731132987867Flushing stores of TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee. at 1731132987868 (+1 ms)Flushing f48863a85e472c28d23fb8419a1993ee/A: creating writer at 1731132987868Flushing mob file f48863a85e472c28d23fb8419a1993ee/A: appending metadata at 1731132987879 (+11 ms)Flushing mob file f48863a85e472c28d23fb8419a1993ee/A: closing flushed file at 1731132987879Flushing f48863a85e472c28d23fb8419a1993ee/A: appending metadata at 1731132988292 (+413 ms)Flushing f48863a85e472c28d23fb8419a1993ee/A: closing flushed file at 1731132988292Flushing f48863a85e472c28d23fb8419a1993ee/B: creating writer at 1731132988702 (+410 ms)Flushing f48863a85e472c28d23fb8419a1993ee/B: appending metadata at 1731132988707 (+5 ms)Flushing f48863a85e472c28d23fb8419a1993ee/B: closing flushed file at 1731132988707Flushing f48863a85e472c28d23fb8419a1993ee/C: creating writer at 1731132989116 (+409 ms)Flushing f48863a85e472c28d23fb8419a1993ee/C: appending metadata at 1731132989119 (+3 ms)Flushing f48863a85e472c28d23fb8419a1993ee/C: closing flushed file at 1731132989119Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28e5b9fe: reopening flushed file at 1731132989527 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23df950e: reopening flushed file at 1731132989533 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2818f547: reopening flushed file at 1731132989537 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f48863a85e472c28d23fb8419a1993ee in 1675ms, sequenceid=467, compaction requested=true at 1731132989543 (+6 ms)Writing region close event to WAL at 1731132989642 (+99 ms)Running coprocessor post-close hooks at 1731132989646 (+4 ms)Closed at 1731132989646 2024-11-09T06:16:29,649 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,649 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=f48863a85e472c28d23fb8419a1993ee, regionState=CLOSED 2024-11-09T06:16:29,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:16:29,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-09T06:16:29,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure f48863a85e472c28d23fb8419a1993ee, server=4a84bcce7bad,46317,1731132929173 in 1.9370 sec 2024-11-09T06:16:29,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-09T06:16:29,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f48863a85e472c28d23fb8419a1993ee, UNASSIGN in 1.9420 sec 2024-11-09T06:16:29,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-09T06:16:29,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9460 sec 2024-11-09T06:16:29,658 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731132989657"}]},"ts":"1731132989657"} 2024-11-09T06:16:29,659 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T06:16:29,659 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T06:16:29,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9550 sec 2024-11-09T06:16:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T06:16:29,853 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:29,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T06:16:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:29,855 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T06:16:29,855 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:29,858 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,861 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/recovered.edits] 2024-11-09T06:16:29,864 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/de1db91542b64987af4aa562716eee1b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/de1db91542b64987af4aa562716eee1b 2024-11-09T06:16:29,865 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/de7479854ba64a68b98f4055c01bca31 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/de7479854ba64a68b98f4055c01bca31 2024-11-09T06:16:29,867 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/eb67e6e7ecf84fce80842c0dc8bf0faf to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/A/eb67e6e7ecf84fce80842c0dc8bf0faf 2024-11-09T06:16:29,869 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/115abf0aa3b24bd28f653bf6add17929 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/115abf0aa3b24bd28f653bf6add17929 2024-11-09T06:16:29,870 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/506fb9f8f58b4b819a1a4b7800ed9d29 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/506fb9f8f58b4b819a1a4b7800ed9d29 2024-11-09T06:16:29,872 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/b498817331d049e29d81023d5b241da4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/B/b498817331d049e29d81023d5b241da4 2024-11-09T06:16:29,874 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/44cd14cae4264d069451a4e90206ad38 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/44cd14cae4264d069451a4e90206ad38 2024-11-09T06:16:29,875 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/d33809fd12c247a2b81c437871fbcfc7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/d33809fd12c247a2b81c437871fbcfc7 2024-11-09T06:16:29,876 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/de7480a2fb4247c1b51f66e609799f8c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/C/de7480a2fb4247c1b51f66e609799f8c 2024-11-09T06:16:29,879 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/recovered.edits/470.seqid to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee/recovered.edits/470.seqid 2024-11-09T06:16:29,879 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,879 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T06:16:29,880 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T06:16:29,881 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T06:16:29,885 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109053e385a385f4a7eb3b9d04d1fec8b52_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109053e385a385f4a7eb3b9d04d1fec8b52_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,886 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090fc52b61fd6d48e4a24967115d5e8dbd_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090fc52b61fd6d48e4a24967115d5e8dbd_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,887 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109189b665d27034e689bda6baf0f1bd4b4_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109189b665d27034e689bda6baf0f1bd4b4_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,888 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093b8364c3fa494781a909c761a200983a_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093b8364c3fa494781a909c761a200983a_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,889 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109442a320377d14d09be16f9e0755d43df_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109442a320377d14d09be16f9e0755d43df_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,890 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109492c7d0f87a840cf883b91132727be76_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109492c7d0f87a840cf883b91132727be76_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,892 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095ad6cb31552545c4a6eb9578cfab6620_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095ad6cb31552545c4a6eb9578cfab6620_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,893 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109614afc8afe9a4367af2ecfa65c50ba11_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109614afc8afe9a4367af2ecfa65c50ba11_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,894 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110961f5a1b26db147e490f4189c1b17c2fa_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110961f5a1b26db147e490f4189c1b17c2fa_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,896 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109685d04882a0c4ae8b92d00d494c62dff_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109685d04882a0c4ae8b92d00d494c62dff_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,897 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097b8f0ec35a114b49b01651530170c789_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097b8f0ec35a114b49b01651530170c789_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,898 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098d5a69563c564298a3362d46ceddfee0_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098d5a69563c564298a3362d46ceddfee0_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,899 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a5d603e404424234881ff61399b5eec4_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a5d603e404424234881ff61399b5eec4_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,900 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a6a6c4c4811f4769b8b2f11507b3a3a3_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a6a6c4c4811f4769b8b2f11507b3a3a3_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,901 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b8e300ea6a904baa8cf15a8c05b765f4_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b8e300ea6a904baa8cf15a8c05b765f4_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,902 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cec84d1afe1f4b91a3d39c0283678c5b_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cec84d1afe1f4b91a3d39c0283678c5b_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,903 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d04cd99b629446189ee90070af259d5c_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d04cd99b629446189ee90070af259d5c_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,904 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109df5d2fbe971a48a9880c823ba0f6f1cc_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109df5d2fbe971a48a9880c823ba0f6f1cc_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,905 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e2a0058b3cd8418d92abcdcd97a1f4a3_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e2a0058b3cd8418d92abcdcd97a1f4a3_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,906 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e6bde1c90e8e4d4e8651352f335b9e2e_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e6bde1c90e8e4d4e8651352f335b9e2e_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,907 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eeb072ab3e6647a9903454886a995544_f48863a85e472c28d23fb8419a1993ee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eeb072ab3e6647a9903454886a995544_f48863a85e472c28d23fb8419a1993ee 2024-11-09T06:16:29,908 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T06:16:29,910 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:29,913 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T06:16:29,915 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T06:16:29,916 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:29,916 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T06:16:29,916 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731132989916"}]},"ts":"9223372036854775807"} 2024-11-09T06:16:29,918 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T06:16:29,918 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f48863a85e472c28d23fb8419a1993ee, NAME => 'TestAcidGuarantees,,1731132961153.f48863a85e472c28d23fb8419a1993ee.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T06:16:29,919 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T06:16:29,919 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731132989919"}]},"ts":"9223372036854775807"} 2024-11-09T06:16:29,921 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T06:16:29,921 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:29,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-09T06:16:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T06:16:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T06:16:29,965 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:29,973 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 248) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/cluster_d5b7a4cc-a1e2-6520-f0a5-179e578abc6c/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043040278_22 at /127.0.0.1:52142 [Waiting for operation #1036] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-864101692_22 at /127.0.0.1:50256 [Waiting for operation #571] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043040278_22 at /127.0.0.1:42988 [Waiting for operation #746] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/cluster_d5b7a4cc-a1e2-6520-f0a5-179e578abc6c/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-864101692_22 at /127.0.0.1:43232 [Waiting for operation #677] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=761 (was 628) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=732 (was 1200) 2024-11-09T06:16:29,986 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=761, ProcessCount=11, AvailableMemoryMB=731 2024-11-09T06:16:29,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T06:16:29,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T06:16:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:29,990 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T06:16:29,990 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:29,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-09T06:16:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-09T06:16:29,992 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T06:16:30,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742064_1240 (size=1061) 2024-11-09T06:16:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-09T06:16:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-09T06:16:30,404 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f 2024-11-09T06:16:30,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742065_1241 (size=53) 2024-11-09T06:16:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-09T06:16:30,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:16:30,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 96ddebd30416e9671cc930feb19be277, disabling compactions & flushes 2024-11-09T06:16:30,815 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:30,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:30,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. after waiting 0 ms 2024-11-09T06:16:30,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:30,815 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:30,815 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 96ddebd30416e9671cc930feb19be277: Waiting for close lock at 1731132990815Disabling compacts and flushes for region at 1731132990815Disabling writes for close at 1731132990815Writing region close event to WAL at 1731132990815Closed at 1731132990815 2024-11-09T06:16:30,816 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T06:16:30,817 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731132990816"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731132990816"}]},"ts":"1731132990816"} 2024-11-09T06:16:30,818 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T06:16:30,819 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T06:16:30,820 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731132990819"}]},"ts":"1731132990819"} 2024-11-09T06:16:30,821 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T06:16:30,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ddebd30416e9671cc930feb19be277, ASSIGN}] 2024-11-09T06:16:30,822 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ddebd30416e9671cc930feb19be277, ASSIGN 2024-11-09T06:16:30,823 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ddebd30416e9671cc930feb19be277, ASSIGN; state=OFFLINE, location=4a84bcce7bad,46317,1731132929173; forceNewPlan=false, retain=false 2024-11-09T06:16:30,973 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=96ddebd30416e9671cc930feb19be277, regionState=OPENING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:30,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ddebd30416e9671cc930feb19be277, ASSIGN because future has completed 2024-11-09T06:16:30,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:16:31,131 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:31,131 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} 2024-11-09T06:16:31,131 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,131 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:16:31,131 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,131 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,133 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-09T06:16:31,134 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:16:31,134 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96ddebd30416e9671cc930feb19be277 columnFamilyName A 2024-11-09T06:16:31,134 DEBUG [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:31,135 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] regionserver.HStore(327): Store=96ddebd30416e9671cc930feb19be277/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:16:31,135 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,136 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:16:31,137 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96ddebd30416e9671cc930feb19be277 columnFamilyName B 2024-11-09T06:16:31,137 DEBUG [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:31,137 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] regionserver.HStore(327): Store=96ddebd30416e9671cc930feb19be277/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:16:31,137 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,138 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:16:31,138 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96ddebd30416e9671cc930feb19be277 columnFamilyName C 2024-11-09T06:16:31,139 DEBUG [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:31,139 INFO [StoreOpener-96ddebd30416e9671cc930feb19be277-1 {}] regionserver.HStore(327): Store=96ddebd30416e9671cc930feb19be277/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:16:31,139 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:31,139 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,140 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,140 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,141 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,141 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,142 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T06:16:31,143 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,145 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T06:16:31,145 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 96ddebd30416e9671cc930feb19be277; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70513543, jitterRate=0.050733670592308044}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T06:16:31,145 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:31,146 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 96ddebd30416e9671cc930feb19be277: Running coprocessor pre-open hook at 1731132991131Writing region info on filesystem at 1731132991131Initializing all the Stores at 1731132991132 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132991132Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132991133 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731132991133Cleaning up temporary data from old regions at 1731132991141 (+8 ms)Running coprocessor post-open hooks at 1731132991145 (+4 ms)Region opened successfully at 1731132991146 (+1 ms) 2024-11-09T06:16:31,147 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., pid=57, masterSystemTime=1731132991127 2024-11-09T06:16:31,149 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:31,149 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:31,150 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=96ddebd30416e9671cc930feb19be277, regionState=OPEN, openSeqNum=2, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:31,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:16:31,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-09T06:16:31,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 in 177 msec 2024-11-09T06:16:31,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-09T06:16:31,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ddebd30416e9671cc930feb19be277, ASSIGN in 333 msec 2024-11-09T06:16:31,157 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T06:16:31,158 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731132991157"}]},"ts":"1731132991157"} 2024-11-09T06:16:31,159 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T06:16:31,160 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T06:16:31,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-09T06:16:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-09T06:16:32,143 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:32,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3260768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:32,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:32,145 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339ca12b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:32,147 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b023a0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,149 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:32,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:32,152 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T06:16:32,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a84bcce7bad,37069,1731132928314 2024-11-09T06:16:32,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7ff0d21a 2024-11-09T06:16:32,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T06:16:32,155 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T06:16:32,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555b0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:32,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:32,157 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f7881c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:32,159 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:32,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba42ecf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:32,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:32,161 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d59b38d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:32,163 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a939b94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:32,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:32,166 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78019d4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:32,167 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6328ddbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:32,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:32,170 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@98a2220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:32,171 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26099430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:32,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:32,174 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b6da7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:32,176 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@505c0be1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:32,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:32,178 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b402dc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:32,180 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:32,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@115715ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:32,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:32,182 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b0b182, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:32,184 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d1b6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:32,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:32,186 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42283f50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:32,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:32,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:32,188 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205bcdcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:32,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:32,190 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4faabe15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:32,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:32,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288732db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:16:32,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:16:32,193 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d997e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:32,195 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:16:32,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d0ac5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:16:32,196 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:32,196 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:32,196 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:32,196 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:32,196 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:32,197 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:32,197 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:32,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:32,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:32,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:32,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:32,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:32,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:32,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:32,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:16:32,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:16:32,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:32,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:16:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:16:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-09T06:16:32,202 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:32,202 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T06:16:32,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:32,203 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:32,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:32,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:16:32,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:32,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:32,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:32,206 DEBUG [Thread-1133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:16:32,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:16:32,207 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:32,207 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:32,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:32,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:16:32,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:16:32,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:16:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:16:32,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:32,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:16:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:16:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:16:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:32,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T06:16:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:32,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/2057bcd49a4140aaab4819ad2867a929 is 50, key is test_row_0/A:col10/1731132992244/Put/seqid=0 2024-11-09T06:16:32,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133052290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133052287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133052292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133052291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133052302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T06:16:32,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742066_1242 (size=12001) 2024-11-09T06:16:32,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/2057bcd49a4140aaab4819ad2867a929 2024-11-09T06:16:32,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T06:16:32,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:32,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:32,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:32,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133052412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133052413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133052414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133052414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133052414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/d1308d300f494e03b5fbc9f481267730 is 50, key is test_row_0/B:col10/1731132992244/Put/seqid=0 2024-11-09T06:16:32,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742067_1243 (size=12001) 2024-11-09T06:16:32,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T06:16:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:32,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T06:16:32,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133052623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133052623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133052624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133052624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133052625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T06:16:32,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:32,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:32,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:32,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T06:16:32,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:32,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:32,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:32,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T06:16:32,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/d1308d300f494e03b5fbc9f481267730 2024-11-09T06:16:32,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/93bd644863204668a46878c8db19b173 is 50, key is test_row_0/C:col10/1731132992244/Put/seqid=0 2024-11-09T06:16:32,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742068_1244 (size=12001) 2024-11-09T06:16:32,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/93bd644863204668a46878c8db19b173 2024-11-09T06:16:32,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133052933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133052933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133052933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/2057bcd49a4140aaab4819ad2867a929 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/2057bcd49a4140aaab4819ad2867a929 2024-11-09T06:16:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133052933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133052934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:32,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/2057bcd49a4140aaab4819ad2867a929, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T06:16:32,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/d1308d300f494e03b5fbc9f481267730 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/d1308d300f494e03b5fbc9f481267730 2024-11-09T06:16:32,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/d1308d300f494e03b5fbc9f481267730, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T06:16:32,975 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-09T06:16:32,975 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-09T06:16:32,980 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-09T06:16:32,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T06:16:32,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:32,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:32,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:32,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:32,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/93bd644863204668a46878c8db19b173 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/93bd644863204668a46878c8db19b173 2024-11-09T06:16:32,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/93bd644863204668a46878c8db19b173, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T06:16:32,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 96ddebd30416e9671cc930feb19be277 in 749ms, sequenceid=13, compaction requested=false 2024-11-09T06:16:32,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:33,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T06:16:33,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:33,135 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:16:33,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:33,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:33,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:33,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:33,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:33,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:33,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/61d92dfc4ae14959b33a3ca1edead806 is 50, key is test_row_0/A:col10/1731132992300/Put/seqid=0 2024-11-09T06:16:33,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742069_1245 (size=12001) 2024-11-09T06:16:33,208 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/61d92dfc4ae14959b33a3ca1edead806 2024-11-09T06:16:33,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/2a7bd850e682431c8ff2fc5cfc046e00 is 50, key is test_row_0/B:col10/1731132992300/Put/seqid=0 2024-11-09T06:16:33,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742070_1246 (size=12001) 2024-11-09T06:16:33,270 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/2a7bd850e682431c8ff2fc5cfc046e00 2024-11-09T06:16:33,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/cee07c607cf44f38b76db494b0ae8f13 is 50, key is test_row_0/C:col10/1731132992300/Put/seqid=0 2024-11-09T06:16:33,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742071_1247 (size=12001) 2024-11-09T06:16:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T06:16:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:33,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:33,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133053451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133053456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133053456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133053458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133053458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133053563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133053563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133053563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133053573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133053574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,738 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/cee07c607cf44f38b76db494b0ae8f13 2024-11-09T06:16:33,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/61d92dfc4ae14959b33a3ca1edead806 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/61d92dfc4ae14959b33a3ca1edead806 2024-11-09T06:16:33,748 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/61d92dfc4ae14959b33a3ca1edead806, entries=150, sequenceid=33, filesize=11.7 K 2024-11-09T06:16:33,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/2a7bd850e682431c8ff2fc5cfc046e00 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/2a7bd850e682431c8ff2fc5cfc046e00 2024-11-09T06:16:33,753 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/2a7bd850e682431c8ff2fc5cfc046e00, entries=150, sequenceid=33, filesize=11.7 K 2024-11-09T06:16:33,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/cee07c607cf44f38b76db494b0ae8f13 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/cee07c607cf44f38b76db494b0ae8f13 2024-11-09T06:16:33,759 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/cee07c607cf44f38b76db494b0ae8f13, entries=150, sequenceid=33, filesize=11.7 K 2024-11-09T06:16:33,760 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 96ddebd30416e9671cc930feb19be277 in 625ms, sequenceid=33, compaction requested=false 2024-11-09T06:16:33,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:33,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:33,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-09T06:16:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-09T06:16:33,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-09T06:16:33,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5580 sec 2024-11-09T06:16:33,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.5670 sec 2024-11-09T06:16:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:33,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:16:33,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:33,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:33,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:33,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:33,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:33,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:33,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/948c8254e1984faba9dc1d0f3c976150 is 50, key is test_row_0/A:col10/1731132993775/Put/seqid=0 2024-11-09T06:16:33,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742072_1248 (size=12001) 2024-11-09T06:16:33,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133053799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133053800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133053800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133053801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133053803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133053913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133053913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133053913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133053914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:33,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133053914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133054123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133054123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133054123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133054124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133054124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/948c8254e1984faba9dc1d0f3c976150 2024-11-09T06:16:34,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/dfcdf1aa2b804e3ba3cfd7298a73b01a is 50, key is test_row_0/B:col10/1731132993775/Put/seqid=0 2024-11-09T06:16:34,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742073_1249 (size=12001) 2024-11-09T06:16:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T06:16:34,353 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:34,353 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-09T06:16:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T06:16:34,356 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:34,357 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:34,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:34,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133054433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133054433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133054433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133054433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133054433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T06:16:34,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T06:16:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:34,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:34,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/dfcdf1aa2b804e3ba3cfd7298a73b01a 2024-11-09T06:16:34,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/59a88d984b6a42aa99be056592e9f815 is 50, key is test_row_0/C:col10/1731132993775/Put/seqid=0 2024-11-09T06:16:34,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742074_1250 (size=12001) 2024-11-09T06:16:34,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/59a88d984b6a42aa99be056592e9f815 2024-11-09T06:16:34,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T06:16:34,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:34,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:34,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:34,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:34,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/948c8254e1984faba9dc1d0f3c976150 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/948c8254e1984faba9dc1d0f3c976150 2024-11-09T06:16:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T06:16:34,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/948c8254e1984faba9dc1d0f3c976150, entries=150, sequenceid=48, filesize=11.7 K 2024-11-09T06:16:34,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/dfcdf1aa2b804e3ba3cfd7298a73b01a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/dfcdf1aa2b804e3ba3cfd7298a73b01a 2024-11-09T06:16:34,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/dfcdf1aa2b804e3ba3cfd7298a73b01a, entries=150, sequenceid=48, filesize=11.7 K 2024-11-09T06:16:34,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/59a88d984b6a42aa99be056592e9f815 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/59a88d984b6a42aa99be056592e9f815 2024-11-09T06:16:34,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/59a88d984b6a42aa99be056592e9f815, entries=150, sequenceid=48, filesize=11.7 K 2024-11-09T06:16:34,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 96ddebd30416e9671cc930feb19be277 in 931ms, sequenceid=48, compaction requested=true 2024-11-09T06:16:34,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:34,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:34,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:34,706 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:34,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:34,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:34,707 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:34,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:34,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:34,708 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:34,708 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/B is initiating minor compaction (all files) 2024-11-09T06:16:34,708 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/B in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:34,708 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/d1308d300f494e03b5fbc9f481267730, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/2a7bd850e682431c8ff2fc5cfc046e00, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/dfcdf1aa2b804e3ba3cfd7298a73b01a] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=35.2 K 2024-11-09T06:16:34,708 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:34,708 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/A is initiating minor compaction (all files) 2024-11-09T06:16:34,708 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/A in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:34,708 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/2057bcd49a4140aaab4819ad2867a929, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/61d92dfc4ae14959b33a3ca1edead806, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/948c8254e1984faba9dc1d0f3c976150] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=35.2 K 2024-11-09T06:16:34,708 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting d1308d300f494e03b5fbc9f481267730, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731132992244 2024-11-09T06:16:34,709 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2057bcd49a4140aaab4819ad2867a929, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731132992244 2024-11-09T06:16:34,709 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a7bd850e682431c8ff2fc5cfc046e00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731132992280 2024-11-09T06:16:34,709 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61d92dfc4ae14959b33a3ca1edead806, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731132992280 2024-11-09T06:16:34,710 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 948c8254e1984faba9dc1d0f3c976150, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731132993456 2024-11-09T06:16:34,710 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting dfcdf1aa2b804e3ba3cfd7298a73b01a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731132993456 2024-11-09T06:16:34,733 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#A#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:34,733 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/845067298f134c37875482090bd39d4f is 50, key is test_row_0/A:col10/1731132993775/Put/seqid=0 2024-11-09T06:16:34,742 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:34,743 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/99d79cdabc18403489c61235873dc3d7 is 50, key is test_row_0/B:col10/1731132993775/Put/seqid=0 2024-11-09T06:16:34,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742075_1251 (size=12104) 2024-11-09T06:16:34,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742076_1252 (size=12104) 2024-11-09T06:16:34,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T06:16:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:34,822 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T06:16:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:34,824 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/99d79cdabc18403489c61235873dc3d7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/99d79cdabc18403489c61235873dc3d7 2024-11-09T06:16:34,831 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/B of 96ddebd30416e9671cc930feb19be277 into 99d79cdabc18403489c61235873dc3d7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:34,831 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:34,831 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/B, priority=13, startTime=1731132994706; duration=0sec 2024-11-09T06:16:34,831 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:34,831 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:B 2024-11-09T06:16:34,831 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:34,833 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:34,833 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/C is initiating minor compaction (all files) 2024-11-09T06:16:34,833 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/C in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:34,833 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/93bd644863204668a46878c8db19b173, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/cee07c607cf44f38b76db494b0ae8f13, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/59a88d984b6a42aa99be056592e9f815] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=35.2 K 2024-11-09T06:16:34,833 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 93bd644863204668a46878c8db19b173, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731132992244 2024-11-09T06:16:34,835 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting cee07c607cf44f38b76db494b0ae8f13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731132992280 2024-11-09T06:16:34,835 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a88d984b6a42aa99be056592e9f815, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731132993456 2024-11-09T06:16:34,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/96d66f018bcb438cbdc2fd8ae0d55f10 is 50, key is test_row_0/A:col10/1731132993799/Put/seqid=0 2024-11-09T06:16:34,851 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#C#compaction#206 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:34,851 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/91449b4bf1514c58abc49d981eb830b0 is 50, key is test_row_0/C:col10/1731132993775/Put/seqid=0 2024-11-09T06:16:34,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742077_1253 (size=12001) 2024-11-09T06:16:34,865 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/96d66f018bcb438cbdc2fd8ae0d55f10 2024-11-09T06:16:34,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742078_1254 (size=12104) 2024-11-09T06:16:34,892 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/91449b4bf1514c58abc49d981eb830b0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/91449b4bf1514c58abc49d981eb830b0 2024-11-09T06:16:34,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/95c5043b30ae490ea1dd92f43008c503 is 50, key is test_row_0/B:col10/1731132993799/Put/seqid=0 2024-11-09T06:16:34,899 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/C of 96ddebd30416e9671cc930feb19be277 into 91449b4bf1514c58abc49d981eb830b0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:34,899 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:34,899 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/C, priority=13, startTime=1731132994707; duration=0sec 2024-11-09T06:16:34,899 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:34,900 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:C 2024-11-09T06:16:34,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742079_1255 (size=12001) 2024-11-09T06:16:34,913 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/95c5043b30ae490ea1dd92f43008c503 2024-11-09T06:16:34,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/0adebb1da0654136b287d2cd4885fa64 is 50, key is test_row_0/C:col10/1731132993799/Put/seqid=0 2024-11-09T06:16:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:34,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:34,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742080_1256 (size=12001) 2024-11-09T06:16:34,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133054967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133054967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133054968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133054969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133054969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T06:16:35,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133055082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133055082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133055082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133055083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133055083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,180 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T06:16:35,216 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/845067298f134c37875482090bd39d4f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/845067298f134c37875482090bd39d4f 2024-11-09T06:16:35,224 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/A of 96ddebd30416e9671cc930feb19be277 into 845067298f134c37875482090bd39d4f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:35,224 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:35,224 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/A, priority=13, startTime=1731132994706; duration=0sec 2024-11-09T06:16:35,224 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:35,224 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:A 2024-11-09T06:16:35,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133055293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133055293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133055293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133055293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133055293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,364 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/0adebb1da0654136b287d2cd4885fa64 2024-11-09T06:16:35,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/96d66f018bcb438cbdc2fd8ae0d55f10 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/96d66f018bcb438cbdc2fd8ae0d55f10 2024-11-09T06:16:35,379 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/96d66f018bcb438cbdc2fd8ae0d55f10, entries=150, sequenceid=65, filesize=11.7 K 2024-11-09T06:16:35,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/95c5043b30ae490ea1dd92f43008c503 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/95c5043b30ae490ea1dd92f43008c503 2024-11-09T06:16:35,388 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/95c5043b30ae490ea1dd92f43008c503, entries=150, sequenceid=65, filesize=11.7 K 2024-11-09T06:16:35,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/0adebb1da0654136b287d2cd4885fa64 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0adebb1da0654136b287d2cd4885fa64 2024-11-09T06:16:35,398 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0adebb1da0654136b287d2cd4885fa64, entries=150, sequenceid=65, filesize=11.7 K 2024-11-09T06:16:35,399 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 96ddebd30416e9671cc930feb19be277 in 577ms, sequenceid=65, compaction requested=false 2024-11-09T06:16:35,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:35,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:35,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-09T06:16:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-09T06:16:35,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-09T06:16:35,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0440 sec 2024-11-09T06:16:35,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.0610 sec 2024-11-09T06:16:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T06:16:35,492 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:35,493 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-09T06:16:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T06:16:35,494 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:35,495 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:35,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T06:16:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:35,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T06:16:35,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:35,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:35,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:35,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:35,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:35,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:35,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/472324552ed5426193acd059f480c815 is 50, key is test_row_0/A:col10/1731132995606/Put/seqid=0 2024-11-09T06:16:35,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742081_1257 (size=14341) 2024-11-09T06:16:35,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133055622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133055623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133055624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133055625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133055625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T06:16:35,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:35,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:35,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:35,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:35,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:35,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133055733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133055733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133055733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133055742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133055742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T06:16:35,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:35,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:35,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:35,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:35,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T06:16:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133055942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133055942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133055942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133055953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133055954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:35,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T06:16:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:35,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:35,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/472324552ed5426193acd059f480c815 2024-11-09T06:16:36,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/3d70a5a2e2b74c7ebdf6c8ca3394404d is 50, key is test_row_0/B:col10/1731132995606/Put/seqid=0 2024-11-09T06:16:36,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742082_1258 (size=12001) 2024-11-09T06:16:36,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T06:16:36,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:36,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T06:16:36,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133056253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133056253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133056253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:36,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133056262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:36,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133056262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:36,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T06:16:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T06:16:36,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:36,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/3d70a5a2e2b74c7ebdf6c8ca3394404d 2024-11-09T06:16:36,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/961326ba0b2c4c5482c806203440a905 is 50, key is test_row_0/C:col10/1731132995606/Put/seqid=0 2024-11-09T06:16:36,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742083_1259 (size=12001) 2024-11-09T06:16:36,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T06:16:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T06:16:36,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T06:16:36,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:36,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133056762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133056763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133056763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:36,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133056773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:36,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133056773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:36,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T06:16:36,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:36,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:36,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/961326ba0b2c4c5482c806203440a905 2024-11-09T06:16:36,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/472324552ed5426193acd059f480c815 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/472324552ed5426193acd059f480c815 2024-11-09T06:16:36,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/472324552ed5426193acd059f480c815, entries=200, sequenceid=86, filesize=14.0 K 2024-11-09T06:16:36,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/3d70a5a2e2b74c7ebdf6c8ca3394404d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/3d70a5a2e2b74c7ebdf6c8ca3394404d 2024-11-09T06:16:36,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/3d70a5a2e2b74c7ebdf6c8ca3394404d, entries=150, sequenceid=86, filesize=11.7 K 2024-11-09T06:16:36,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/961326ba0b2c4c5482c806203440a905 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/961326ba0b2c4c5482c806203440a905 2024-11-09T06:16:36,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/961326ba0b2c4c5482c806203440a905, entries=150, sequenceid=86, filesize=11.7 K 2024-11-09T06:16:36,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 96ddebd30416e9671cc930feb19be277 in 1345ms, sequenceid=86, compaction requested=true 2024-11-09T06:16:36,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:36,951 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:36,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:36,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:36,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:36,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:36,952 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:36,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:36,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:36,952 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:36,952 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/A is initiating minor compaction (all files) 2024-11-09T06:16:36,952 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/A in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,952 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/845067298f134c37875482090bd39d4f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/96d66f018bcb438cbdc2fd8ae0d55f10, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/472324552ed5426193acd059f480c815] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=37.5 K 2024-11-09T06:16:36,953 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 845067298f134c37875482090bd39d4f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731132993456 2024-11-09T06:16:36,953 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:36,953 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/B is initiating minor compaction (all files) 2024-11-09T06:16:36,953 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/B in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:36,953 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96d66f018bcb438cbdc2fd8ae0d55f10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731132993799 2024-11-09T06:16:36,953 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/99d79cdabc18403489c61235873dc3d7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/95c5043b30ae490ea1dd92f43008c503, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/3d70a5a2e2b74c7ebdf6c8ca3394404d] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=35.3 K 2024-11-09T06:16:36,954 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 472324552ed5426193acd059f480c815, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731132994966 2024-11-09T06:16:36,954 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 99d79cdabc18403489c61235873dc3d7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731132993456 2024-11-09T06:16:36,955 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 95c5043b30ae490ea1dd92f43008c503, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731132993799 2024-11-09T06:16:36,955 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d70a5a2e2b74c7ebdf6c8ca3394404d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731132994967 2024-11-09T06:16:36,967 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#B#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:36,967 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/4b29cffa2cc84974924fc7d7bd51a579 is 50, key is test_row_0/B:col10/1731132995606/Put/seqid=0 2024-11-09T06:16:36,973 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#A#compaction#213 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:36,973 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/18dc9a1c05f8475b90d09abef2bd866b is 50, key is test_row_0/A:col10/1731132995606/Put/seqid=0 2024-11-09T06:16:36,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742085_1261 (size=12207) 2024-11-09T06:16:36,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742084_1260 (size=12207) 2024-11-09T06:16:37,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T06:16:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:37,038 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T06:16:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:37,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:37,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:37,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:37,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:37,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f7ec945462ef4e74ad26dac4421b6006 is 50, key is test_row_0/A:col10/1731132995622/Put/seqid=0 2024-11-09T06:16:37,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742086_1262 (size=12001) 2024-11-09T06:16:37,075 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f7ec945462ef4e74ad26dac4421b6006 2024-11-09T06:16:37,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/46dbcd475a1449f98dc0dfe2886c1efb is 50, key is test_row_0/B:col10/1731132995622/Put/seqid=0 2024-11-09T06:16:37,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742087_1263 (size=12001) 2024-11-09T06:16:37,133 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/46dbcd475a1449f98dc0dfe2886c1efb 2024-11-09T06:16:37,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/20b43f9659bf44e69113363dce0df27f is 50, key is test_row_0/C:col10/1731132995622/Put/seqid=0 2024-11-09T06:16:37,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742088_1264 (size=12001) 2024-11-09T06:16:37,391 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/18dc9a1c05f8475b90d09abef2bd866b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/18dc9a1c05f8475b90d09abef2bd866b 2024-11-09T06:16:37,394 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/4b29cffa2cc84974924fc7d7bd51a579 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4b29cffa2cc84974924fc7d7bd51a579 2024-11-09T06:16:37,400 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/A of 96ddebd30416e9671cc930feb19be277 into 18dc9a1c05f8475b90d09abef2bd866b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:37,401 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:37,401 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/A, priority=13, startTime=1731132996951; duration=0sec 2024-11-09T06:16:37,401 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:37,401 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:A 2024-11-09T06:16:37,401 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:37,403 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/B of 96ddebd30416e9671cc930feb19be277 into 4b29cffa2cc84974924fc7d7bd51a579(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:37,403 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:37,403 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/B, priority=13, startTime=1731132996952; duration=0sec 2024-11-09T06:16:37,404 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:37,404 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:B 2024-11-09T06:16:37,404 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:37,404 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/C is initiating minor compaction (all files) 2024-11-09T06:16:37,405 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/C in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:37,405 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/91449b4bf1514c58abc49d981eb830b0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0adebb1da0654136b287d2cd4885fa64, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/961326ba0b2c4c5482c806203440a905] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=35.3 K 2024-11-09T06:16:37,405 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91449b4bf1514c58abc49d981eb830b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731132993456 2024-11-09T06:16:37,406 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0adebb1da0654136b287d2cd4885fa64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731132993799 2024-11-09T06:16:37,406 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 961326ba0b2c4c5482c806203440a905, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731132994967 2024-11-09T06:16:37,419 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#C#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:37,419 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/988e3130156e4897b2f346e14af59d1d is 50, key is test_row_0/C:col10/1731132995606/Put/seqid=0 2024-11-09T06:16:37,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742089_1265 (size=12207) 2024-11-09T06:16:37,462 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/988e3130156e4897b2f346e14af59d1d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/988e3130156e4897b2f346e14af59d1d 2024-11-09T06:16:37,469 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/C of 96ddebd30416e9671cc930feb19be277 into 988e3130156e4897b2f346e14af59d1d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:37,469 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:37,469 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/C, priority=13, startTime=1731132996952; duration=0sec 2024-11-09T06:16:37,469 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:37,469 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:C 2024-11-09T06:16:37,605 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/20b43f9659bf44e69113363dce0df27f 2024-11-09T06:16:37,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f7ec945462ef4e74ad26dac4421b6006 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f7ec945462ef4e74ad26dac4421b6006 2024-11-09T06:16:37,629 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f7ec945462ef4e74ad26dac4421b6006, entries=150, sequenceid=101, filesize=11.7 K 2024-11-09T06:16:37,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/46dbcd475a1449f98dc0dfe2886c1efb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/46dbcd475a1449f98dc0dfe2886c1efb 2024-11-09T06:16:37,636 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/46dbcd475a1449f98dc0dfe2886c1efb, entries=150, sequenceid=101, filesize=11.7 K 2024-11-09T06:16:37,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/20b43f9659bf44e69113363dce0df27f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/20b43f9659bf44e69113363dce0df27f 2024-11-09T06:16:37,647 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/20b43f9659bf44e69113363dce0df27f, entries=150, sequenceid=101, filesize=11.7 K 2024-11-09T06:16:37,648 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 96ddebd30416e9671cc930feb19be277 in 610ms, sequenceid=101, compaction requested=false 2024-11-09T06:16:37,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:37,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:37,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-09T06:16:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-09T06:16:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T06:16:37,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-09T06:16:37,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1560 sec 2024-11-09T06:16:37,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.1660 sec 2024-11-09T06:16:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:37,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:16:37,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:37,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/b0c98206f0494b89b3174f8b64ac0329 is 50, key is test_row_0/A:col10/1731132997786/Put/seqid=0 2024-11-09T06:16:37,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742090_1266 (size=14341) 2024-11-09T06:16:37,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/b0c98206f0494b89b3174f8b64ac0329 2024-11-09T06:16:37,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:37,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133057813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133057811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:37,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133057815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133057816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:37,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133057820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:37,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/4b1f5361b4044da0a96ada6fa8401ef9 is 50, key is test_row_0/B:col10/1731132997786/Put/seqid=0 2024-11-09T06:16:37,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742091_1267 (size=12001) 2024-11-09T06:16:37,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/4b1f5361b4044da0a96ada6fa8401ef9 2024-11-09T06:16:37,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/7bc56cb3c5e34103ba08187c95fe16e8 is 50, key is test_row_0/C:col10/1731132997786/Put/seqid=0 2024-11-09T06:16:37,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742092_1268 (size=12001) 2024-11-09T06:16:37,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133057932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:37,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133057932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133057932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133057933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:37,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133057933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:37,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:37,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:37,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133058143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133058143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133058143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133058143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133058143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/7bc56cb3c5e34103ba08187c95fe16e8 2024-11-09T06:16:38,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/b0c98206f0494b89b3174f8b64ac0329 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/b0c98206f0494b89b3174f8b64ac0329 2024-11-09T06:16:38,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/b0c98206f0494b89b3174f8b64ac0329, entries=200, sequenceid=116, filesize=14.0 K 2024-11-09T06:16:38,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/4b1f5361b4044da0a96ada6fa8401ef9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4b1f5361b4044da0a96ada6fa8401ef9 2024-11-09T06:16:38,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4b1f5361b4044da0a96ada6fa8401ef9, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T06:16:38,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/7bc56cb3c5e34103ba08187c95fe16e8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/7bc56cb3c5e34103ba08187c95fe16e8 2024-11-09T06:16:38,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/7bc56cb3c5e34103ba08187c95fe16e8, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T06:16:38,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 96ddebd30416e9671cc930feb19be277 in 564ms, sequenceid=116, compaction requested=true 2024-11-09T06:16:38,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:38,350 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:38,351 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:38,351 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:38,351 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/A is initiating minor compaction (all files) 2024-11-09T06:16:38,351 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/A in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:38,351 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/18dc9a1c05f8475b90d09abef2bd866b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f7ec945462ef4e74ad26dac4421b6006, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/b0c98206f0494b89b3174f8b64ac0329] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=37.6 K 2024-11-09T06:16:38,352 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:38,352 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/B is initiating minor compaction (all files) 2024-11-09T06:16:38,352 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/B in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:38,352 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4b29cffa2cc84974924fc7d7bd51a579, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/46dbcd475a1449f98dc0dfe2886c1efb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4b1f5361b4044da0a96ada6fa8401ef9] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=35.4 K 2024-11-09T06:16:38,352 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18dc9a1c05f8475b90d09abef2bd866b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731132994967 2024-11-09T06:16:38,353 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b29cffa2cc84974924fc7d7bd51a579, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731132994967 2024-11-09T06:16:38,353 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7ec945462ef4e74ad26dac4421b6006, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731132995612 2024-11-09T06:16:38,353 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 46dbcd475a1449f98dc0dfe2886c1efb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731132995612 2024-11-09T06:16:38,353 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0c98206f0494b89b3174f8b64ac0329, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731132997782 2024-11-09T06:16:38,354 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b1f5361b4044da0a96ada6fa8401ef9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731132997782 2024-11-09T06:16:38,368 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#B#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:38,369 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/95ab49ff525f4fdd8ec7b26cbd170f16 is 50, key is test_row_0/B:col10/1731132997786/Put/seqid=0 2024-11-09T06:16:38,369 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#A#compaction#222 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:38,370 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/485e0936e9d94c00a80877835f2088fa is 50, key is test_row_0/A:col10/1731132997786/Put/seqid=0 2024-11-09T06:16:38,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742093_1269 (size=12309) 2024-11-09T06:16:38,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742094_1270 (size=12309) 2024-11-09T06:16:38,416 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/95ab49ff525f4fdd8ec7b26cbd170f16 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/95ab49ff525f4fdd8ec7b26cbd170f16 2024-11-09T06:16:38,421 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/B of 96ddebd30416e9671cc930feb19be277 into 95ab49ff525f4fdd8ec7b26cbd170f16(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:38,421 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:38,421 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/B, priority=13, startTime=1731132998351; duration=0sec 2024-11-09T06:16:38,421 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:38,421 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:B 2024-11-09T06:16:38,422 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:38,423 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:38,423 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/C is initiating minor compaction (all files) 2024-11-09T06:16:38,423 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/C in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:38,423 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/988e3130156e4897b2f346e14af59d1d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/20b43f9659bf44e69113363dce0df27f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/7bc56cb3c5e34103ba08187c95fe16e8] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=35.4 K 2024-11-09T06:16:38,424 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 988e3130156e4897b2f346e14af59d1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731132994967 2024-11-09T06:16:38,424 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b43f9659bf44e69113363dce0df27f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731132995612 2024-11-09T06:16:38,424 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bc56cb3c5e34103ba08187c95fe16e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731132997782 2024-11-09T06:16:38,432 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#C#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:38,433 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/47d87e07d88142c48a1a8257864cb34a is 50, key is test_row_0/C:col10/1731132997786/Put/seqid=0 2024-11-09T06:16:38,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742095_1271 (size=12309) 2024-11-09T06:16:38,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-09T06:16:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:38,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132997812, max timestamp=1731132998454Num uniques 859; 2024-11-09T06:16:38,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:38,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731132997812, max timestamp=1731132998454Num uniques 859; 2024-11-09T06:16:38,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:38,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731132997812, max timestamp=1731132998454Num uniques 859; 2024-11-09T06:16:38,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-09T06:16:38,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:38,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:38,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:38,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:38,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:38,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:38,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/74c87f5d71e548868a2f7cf58c8789c8 is 50, key is test_row_0/A:col10/1731132998456/Put/seqid=0 2024-11-09T06:16:38,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731132998469, max timestamp=1731132998482Num uniques 859; 2024-11-09T06:16:38,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:38,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731132998469, max timestamp=1731132998482Num uniques 859; 2024-11-09T06:16:38,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731132998469, max timestamp=1731132998482Num uniques 859; 2024-11-09T06:16:38,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:38,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:38,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742096_1272 (size=12151) 2024-11-09T06:16:38,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/74c87f5d71e548868a2f7cf58c8789c8 2024-11-09T06:16:38,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/67ef569a0a1943dc84f22c4fbf08796e is 50, key is test_row_0/B:col10/1731132998456/Put/seqid=0 2024-11-09T06:16:38,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133058508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731132998501, max timestamp=1731132998516Num uniques 1118; 2024-11-09T06:16:38,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-09T06:16:38,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731132998512, max timestamp=1731132998516Num uniques 1118; 2024-11-09T06:16:38,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-09T06:16:38,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731132998512, max timestamp=1731132998516Num uniques 1118; 2024-11-09T06:16:38,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-09T06:16:38,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742097_1273 (size=12151) 2024-11-09T06:16:38,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/67ef569a0a1943dc84f22c4fbf08796e 2024-11-09T06:16:38,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/6c276864377342f383ec10801c1d1b52 is 50, key is test_row_0/C:col10/1731132998456/Put/seqid=0 2024-11-09T06:16:38,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742098_1274 (size=12151) 2024-11-09T06:16:38,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/6c276864377342f383ec10801c1d1b52 2024-11-09T06:16:38,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133058550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133058556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731132998546, max timestamp=1731132998559Num uniques 1027; 2024-11-09T06:16:38,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-09T06:16:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/74c87f5d71e548868a2f7cf58c8789c8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/74c87f5d71e548868a2f7cf58c8789c8 2024-11-09T06:16:38,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731132998546, max timestamp=1731132998559Num uniques 1027; 2024-11-09T06:16:38,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-09T06:16:38,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731132998546, max timestamp=1731132998559Num uniques 1177; 2024-11-09T06:16:38,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-09T06:16:38,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/74c87f5d71e548868a2f7cf58c8789c8, entries=150, sequenceid=140, filesize=11.9 K 2024-11-09T06:16:38,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/67ef569a0a1943dc84f22c4fbf08796e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/67ef569a0a1943dc84f22c4fbf08796e 2024-11-09T06:16:38,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/67ef569a0a1943dc84f22c4fbf08796e, entries=150, sequenceid=140, filesize=11.9 K 2024-11-09T06:16:38,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/6c276864377342f383ec10801c1d1b52 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/6c276864377342f383ec10801c1d1b52 2024-11-09T06:16:38,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/6c276864377342f383ec10801c1d1b52, entries=150, sequenceid=140, filesize=11.9 K 2024-11-09T06:16:38,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.86 KB/40815, heapSize ~105.12 KB/107640, currentSize=73.80 KB/75570 for 96ddebd30416e9671cc930feb19be277 in 144ms, sequenceid=140, compaction requested=false 2024-11-09T06:16:38,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:38,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=80.51 KB heapSize=219.23 KB 2024-11-09T06:16:38,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:38,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:38,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:38,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:38,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:38,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:38,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f0f61ee457fa47419490b7c30011d8d0 is 50, key is test_row_0/A:col10/1731132998603/Put/seqid=0 2024-11-09T06:16:38,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742099_1275 (size=12151) 2024-11-09T06:16:38,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f0f61ee457fa47419490b7c30011d8d0 2024-11-09T06:16:38,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/01c2cc7ba1e541b4a3ad37c994282746 is 50, key is test_row_0/B:col10/1731132998603/Put/seqid=0 2024-11-09T06:16:38,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133058645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133058648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133058652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742100_1276 (size=12151) 2024-11-09T06:16:38,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133058673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133058673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133058763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133058763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133058763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,820 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/485e0936e9d94c00a80877835f2088fa as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/485e0936e9d94c00a80877835f2088fa 2024-11-09T06:16:38,826 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/A of 96ddebd30416e9671cc930feb19be277 into 485e0936e9d94c00a80877835f2088fa(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:38,826 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:38,827 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/A, priority=13, startTime=1731132998350; duration=0sec 2024-11-09T06:16:38,827 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:38,827 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:A 2024-11-09T06:16:38,845 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/47d87e07d88142c48a1a8257864cb34a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/47d87e07d88142c48a1a8257864cb34a 2024-11-09T06:16:38,850 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/C of 96ddebd30416e9671cc930feb19be277 into 47d87e07d88142c48a1a8257864cb34a(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:38,850 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:38,850 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/C, priority=13, startTime=1731132998351; duration=0sec 2024-11-09T06:16:38,851 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:38,851 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:C 2024-11-09T06:16:38,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133058882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133058882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133058972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133058972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133058972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/01c2cc7ba1e541b4a3ad37c994282746 2024-11-09T06:16:39,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/c276957f36d84d03a10b7525d760d15a is 50, key is test_row_0/C:col10/1731132998603/Put/seqid=0 2024-11-09T06:16:39,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742101_1277 (size=12151) 2024-11-09T06:16:39,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/c276957f36d84d03a10b7525d760d15a 2024-11-09T06:16:39,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f0f61ee457fa47419490b7c30011d8d0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f0f61ee457fa47419490b7c30011d8d0 2024-11-09T06:16:39,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f0f61ee457fa47419490b7c30011d8d0, entries=150, sequenceid=203, filesize=11.9 K 2024-11-09T06:16:39,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/01c2cc7ba1e541b4a3ad37c994282746 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/01c2cc7ba1e541b4a3ad37c994282746 2024-11-09T06:16:39,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/01c2cc7ba1e541b4a3ad37c994282746, entries=150, sequenceid=203, filesize=11.9 K 2024-11-09T06:16:39,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/c276957f36d84d03a10b7525d760d15a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/c276957f36d84d03a10b7525d760d15a 2024-11-09T06:16:39,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/c276957f36d84d03a10b7525d760d15a, entries=150, sequenceid=203, filesize=11.9 K 2024-11-09T06:16:39,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~239.11 KB/244848, currentSize=93.93 KB/96180 for 96ddebd30416e9671cc930feb19be277 in 546ms, sequenceid=203, compaction requested=true 2024-11-09T06:16:39,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:39,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:39,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:39,149 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:39,149 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:39,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:39,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:39,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:39,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:39,150 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:39,151 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/B is initiating minor compaction (all files) 2024-11-09T06:16:39,151 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/B in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:39,151 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/95ab49ff525f4fdd8ec7b26cbd170f16, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/67ef569a0a1943dc84f22c4fbf08796e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/01c2cc7ba1e541b4a3ad37c994282746] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=35.8 K 2024-11-09T06:16:39,151 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:39,151 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ab49ff525f4fdd8ec7b26cbd170f16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731132997782 2024-11-09T06:16:39,151 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/A is initiating minor compaction (all files) 2024-11-09T06:16:39,151 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/A in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:39,152 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/485e0936e9d94c00a80877835f2088fa, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/74c87f5d71e548868a2f7cf58c8789c8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f0f61ee457fa47419490b7c30011d8d0] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=35.8 K 2024-11-09T06:16:39,152 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 67ef569a0a1943dc84f22c4fbf08796e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731132997812 2024-11-09T06:16:39,152 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 485e0936e9d94c00a80877835f2088fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731132997782 2024-11-09T06:16:39,152 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 01c2cc7ba1e541b4a3ad37c994282746, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731132998582 2024-11-09T06:16:39,152 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74c87f5d71e548868a2f7cf58c8789c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731132997812 2024-11-09T06:16:39,153 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f61ee457fa47419490b7c30011d8d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731132998582 2024-11-09T06:16:39,178 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#B#compaction#230 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:39,187 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#A#compaction#231 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:39,188 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/4c8fc94a2ac14923bab920c44ebf89f1 is 50, key is test_row_0/B:col10/1731132998603/Put/seqid=0 2024-11-09T06:16:39,192 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/bf85284361ce42778b050ae297691f4e is 50, key is test_row_0/A:col10/1731132998603/Put/seqid=0 2024-11-09T06:16:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:39,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T06:16:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:39,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:39,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:39,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742102_1278 (size=12561) 2024-11-09T06:16:39,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742103_1279 (size=12561) 2024-11-09T06:16:39,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/30b11434204543649310ac1285a3bf10 is 50, key is test_row_0/A:col10/1731132999199/Put/seqid=0 2024-11-09T06:16:39,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742104_1280 (size=14541) 2024-11-09T06:16:39,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/30b11434204543649310ac1285a3bf10 2024-11-09T06:16:39,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133059283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133059283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133059284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/1a6c8fad6c174397b6db9e56df835e06 is 50, key is test_row_0/B:col10/1731132999199/Put/seqid=0 2024-11-09T06:16:39,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133059289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133059296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742105_1281 (size=12151) 2024-11-09T06:16:39,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/1a6c8fad6c174397b6db9e56df835e06 2024-11-09T06:16:39,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/109a85dd78234e9680e98b0ea7f77d85 is 50, key is test_row_0/C:col10/1731132999199/Put/seqid=0 2024-11-09T06:16:39,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742106_1282 (size=12151) 2024-11-09T06:16:39,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133059393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133059402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133059402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133059603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133059612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133059612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,634 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/4c8fc94a2ac14923bab920c44ebf89f1 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4c8fc94a2ac14923bab920c44ebf89f1 2024-11-09T06:16:39,639 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/B of 96ddebd30416e9671cc930feb19be277 into 4c8fc94a2ac14923bab920c44ebf89f1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:39,639 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:39,639 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/B, priority=13, startTime=1731132999149; duration=0sec 2024-11-09T06:16:39,639 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:39,639 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:B 2024-11-09T06:16:39,639 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:39,644 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:39,644 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/C is initiating minor compaction (all files) 2024-11-09T06:16:39,644 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/C in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:39,644 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/47d87e07d88142c48a1a8257864cb34a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/6c276864377342f383ec10801c1d1b52, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/c276957f36d84d03a10b7525d760d15a] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=35.8 K 2024-11-09T06:16:39,645 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d87e07d88142c48a1a8257864cb34a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731132997782 2024-11-09T06:16:39,645 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c276864377342f383ec10801c1d1b52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731132997812 2024-11-09T06:16:39,645 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting c276957f36d84d03a10b7525d760d15a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731132998582 2024-11-09T06:16:39,649 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/bf85284361ce42778b050ae297691f4e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/bf85284361ce42778b050ae297691f4e 2024-11-09T06:16:39,655 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#C#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:39,656 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/A of 96ddebd30416e9671cc930feb19be277 into bf85284361ce42778b050ae297691f4e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:39,656 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:39,656 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/A, priority=13, startTime=1731132999149; duration=0sec 2024-11-09T06:16:39,656 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/8d4beafa00fd48f8b102109a955baa29 is 50, key is test_row_0/C:col10/1731132998603/Put/seqid=0 2024-11-09T06:16:39,656 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:39,656 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:A 2024-11-09T06:16:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T06:16:39,673 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:39,673 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-09T06:16:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T06:16:39,676 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:39,677 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:39,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:39,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742107_1283 (size=12561) 2024-11-09T06:16:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T06:16:39,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/109a85dd78234e9680e98b0ea7f77d85 2024-11-09T06:16:39,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133059792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/30b11434204543649310ac1285a3bf10 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/30b11434204543649310ac1285a3bf10 2024-11-09T06:16:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133059803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/30b11434204543649310ac1285a3bf10, entries=200, sequenceid=224, filesize=14.2 K 2024-11-09T06:16:39,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/1a6c8fad6c174397b6db9e56df835e06 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/1a6c8fad6c174397b6db9e56df835e06 2024-11-09T06:16:39,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/1a6c8fad6c174397b6db9e56df835e06, entries=150, sequenceid=224, filesize=11.9 K 2024-11-09T06:16:39,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/109a85dd78234e9680e98b0ea7f77d85 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/109a85dd78234e9680e98b0ea7f77d85 2024-11-09T06:16:39,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/109a85dd78234e9680e98b0ea7f77d85, entries=150, sequenceid=224, filesize=11.9 K 2024-11-09T06:16:39,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 96ddebd30416e9671cc930feb19be277 in 620ms, sequenceid=224, compaction requested=false 2024-11-09T06:16:39,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:39,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T06:16:39,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:39,830 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T06:16:39,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:39,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:39,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:39,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:39,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:39,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:39,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/c5218a989fbe4c5786e566846ee228c7 is 50, key is test_row_0/A:col10/1731132999287/Put/seqid=0 2024-11-09T06:16:39,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742108_1284 (size=12151) 2024-11-09T06:16:39,855 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/c5218a989fbe4c5786e566846ee228c7 2024-11-09T06:16:39,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/235c2af6b37b4f31a73c8f654dc6eaac is 50, key is test_row_0/B:col10/1731132999287/Put/seqid=0 2024-11-09T06:16:39,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:39,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742109_1285 (size=12151) 2024-11-09T06:16:39,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133059955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133059958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133059959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T06:16:40,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133060063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133060073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133060073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,092 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/8d4beafa00fd48f8b102109a955baa29 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/8d4beafa00fd48f8b102109a955baa29 2024-11-09T06:16:40,103 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/C of 96ddebd30416e9671cc930feb19be277 into 8d4beafa00fd48f8b102109a955baa29(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:40,103 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:40,103 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/C, priority=13, startTime=1731132999149; duration=0sec 2024-11-09T06:16:40,103 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:40,103 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:C 2024-11-09T06:16:40,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133060287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133060287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133060288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T06:16:40,318 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/235c2af6b37b4f31a73c8f654dc6eaac 2024-11-09T06:16:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/75314851d46a4a0b8d71d9eb9c96dff5 is 50, key is test_row_0/C:col10/1731132999287/Put/seqid=0 2024-11-09T06:16:40,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742110_1286 (size=12151) 2024-11-09T06:16:40,330 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/75314851d46a4a0b8d71d9eb9c96dff5 2024-11-09T06:16:40,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/c5218a989fbe4c5786e566846ee228c7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/c5218a989fbe4c5786e566846ee228c7 2024-11-09T06:16:40,342 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/c5218a989fbe4c5786e566846ee228c7, entries=150, sequenceid=240, filesize=11.9 K 2024-11-09T06:16:40,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/235c2af6b37b4f31a73c8f654dc6eaac as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/235c2af6b37b4f31a73c8f654dc6eaac 2024-11-09T06:16:40,351 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/235c2af6b37b4f31a73c8f654dc6eaac, entries=150, sequenceid=240, filesize=11.9 K 2024-11-09T06:16:40,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/75314851d46a4a0b8d71d9eb9c96dff5 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/75314851d46a4a0b8d71d9eb9c96dff5 2024-11-09T06:16:40,356 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/75314851d46a4a0b8d71d9eb9c96dff5, entries=150, sequenceid=240, filesize=11.9 K 2024-11-09T06:16:40,357 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 96ddebd30416e9671cc930feb19be277 in 527ms, sequenceid=240, compaction requested=true 2024-11-09T06:16:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-09T06:16:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-09T06:16:40,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-09T06:16:40,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 685 msec 2024-11-09T06:16:40,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 692 msec 2024-11-09T06:16:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:40,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T06:16:40,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:40,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:40,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:40,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:40,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:40,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:40,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/6482bbca62db452a9be76249f279eaa1 is 50, key is test_row_0/A:col10/1731132999948/Put/seqid=0 2024-11-09T06:16:40,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742111_1287 (size=12251) 2024-11-09T06:16:40,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133060625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133060632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133060632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133060743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133060743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133060743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133060803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,805 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in 96ddebd30416e9671cc930feb19be277 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:16:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T06:16:40,814 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:40,814 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-09T06:16:40,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133060813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T06:16:40,819 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:40,818 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in 96ddebd30416e9671cc930feb19be277 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:16:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,820 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:40,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T06:16:40,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133060953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133060953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133060953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:40,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T06:16:40,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:40,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:40,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:40,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:40,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:41,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/6482bbca62db452a9be76249f279eaa1 2024-11-09T06:16:41,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/5ea981ef49174b95887870e6f195455b is 50, key is test_row_0/B:col10/1731132999948/Put/seqid=0 2024-11-09T06:16:41,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742112_1288 (size=12251) 2024-11-09T06:16:41,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/5ea981ef49174b95887870e6f195455b 2024-11-09T06:16:41,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/5941b714fd694d74a7590caa067e7020 is 50, key is test_row_0/C:col10/1731132999948/Put/seqid=0 2024-11-09T06:16:41,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742113_1289 (size=12251) 2024-11-09T06:16:41,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/5941b714fd694d74a7590caa067e7020 2024-11-09T06:16:41,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/6482bbca62db452a9be76249f279eaa1 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6482bbca62db452a9be76249f279eaa1 2024-11-09T06:16:41,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6482bbca62db452a9be76249f279eaa1, entries=150, sequenceid=260, filesize=12.0 K 2024-11-09T06:16:41,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/5ea981ef49174b95887870e6f195455b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/5ea981ef49174b95887870e6f195455b 2024-11-09T06:16:41,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/5ea981ef49174b95887870e6f195455b, entries=150, sequenceid=260, filesize=12.0 K 2024-11-09T06:16:41,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/5941b714fd694d74a7590caa067e7020 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/5941b714fd694d74a7590caa067e7020 2024-11-09T06:16:41,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/5941b714fd694d74a7590caa067e7020, entries=150, sequenceid=260, filesize=12.0 K 2024-11-09T06:16:41,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 96ddebd30416e9671cc930feb19be277 in 498ms, sequenceid=260, compaction requested=true 2024-11-09T06:16:41,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:41,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:41,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:41,105 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:16:41,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:41,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:41,105 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:16:41,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:41,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:41,106 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49114 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:16:41,107 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/B is initiating minor compaction (all files) 2024-11-09T06:16:41,107 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51504 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:16:41,107 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/A is initiating minor compaction (all files) 2024-11-09T06:16:41,107 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/A in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:41,107 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/bf85284361ce42778b050ae297691f4e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/30b11434204543649310ac1285a3bf10, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/c5218a989fbe4c5786e566846ee228c7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6482bbca62db452a9be76249f279eaa1] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=50.3 K 2024-11-09T06:16:41,108 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/B in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:41,108 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4c8fc94a2ac14923bab920c44ebf89f1, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/1a6c8fad6c174397b6db9e56df835e06, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/235c2af6b37b4f31a73c8f654dc6eaac, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/5ea981ef49174b95887870e6f195455b] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=48.0 K 2024-11-09T06:16:41,108 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf85284361ce42778b050ae297691f4e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731132998582 2024-11-09T06:16:41,108 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c8fc94a2ac14923bab920c44ebf89f1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731132998582 2024-11-09T06:16:41,109 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30b11434204543649310ac1285a3bf10, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731132998626 2024-11-09T06:16:41,109 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a6c8fad6c174397b6db9e56df835e06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731132998637 2024-11-09T06:16:41,109 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5218a989fbe4c5786e566846ee228c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731132999277 2024-11-09T06:16:41,109 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 235c2af6b37b4f31a73c8f654dc6eaac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731132999277 2024-11-09T06:16:41,110 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6482bbca62db452a9be76249f279eaa1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731132999948 2024-11-09T06:16:41,110 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ea981ef49174b95887870e6f195455b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731132999948 2024-11-09T06:16:41,124 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#B#compaction#242 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:41,124 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/82665f34d08f4cd5aa6c6d83d28aa35f is 50, key is test_row_0/B:col10/1731132999948/Put/seqid=0 2024-11-09T06:16:41,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T06:16:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:41,130 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T06:16:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T06:16:41,136 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#A#compaction#243 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:41,137 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/72ad73e04f3945028e0bfa39987d8915 is 50, key is test_row_0/A:col10/1731132999948/Put/seqid=0 2024-11-09T06:16:41,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/0f2b9d70507c448aa4a3c5d2b351fab9 is 50, key is test_row_0/A:col10/1731133000630/Put/seqid=0 2024-11-09T06:16:41,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742114_1290 (size=12797) 2024-11-09T06:16:41,151 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/82665f34d08f4cd5aa6c6d83d28aa35f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/82665f34d08f4cd5aa6c6d83d28aa35f 2024-11-09T06:16:41,161 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ddebd30416e9671cc930feb19be277/B of 96ddebd30416e9671cc930feb19be277 into 82665f34d08f4cd5aa6c6d83d28aa35f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:41,161 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:41,161 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/B, priority=12, startTime=1731133001105; duration=0sec 2024-11-09T06:16:41,161 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:41,161 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:B 2024-11-09T06:16:41,161 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:16:41,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742116_1292 (size=12301) 2024-11-09T06:16:41,165 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/0f2b9d70507c448aa4a3c5d2b351fab9 2024-11-09T06:16:41,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742115_1291 (size=12797) 2024-11-09T06:16:41,167 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49114 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:16:41,167 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/C is initiating minor compaction (all files) 2024-11-09T06:16:41,167 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/C in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:41,168 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/8d4beafa00fd48f8b102109a955baa29, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/109a85dd78234e9680e98b0ea7f77d85, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/75314851d46a4a0b8d71d9eb9c96dff5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/5941b714fd694d74a7590caa067e7020] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=48.0 K 2024-11-09T06:16:41,169 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d4beafa00fd48f8b102109a955baa29, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731132998582 2024-11-09T06:16:41,170 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 109a85dd78234e9680e98b0ea7f77d85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731132998637 2024-11-09T06:16:41,170 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 75314851d46a4a0b8d71d9eb9c96dff5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731132999277 2024-11-09T06:16:41,171 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5941b714fd694d74a7590caa067e7020, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731132999948 2024-11-09T06:16:41,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/ed017d7e412a4ecbb04e49ba6f158f58 is 50, key is test_row_0/B:col10/1731133000630/Put/seqid=0 2024-11-09T06:16:41,192 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#C#compaction#246 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:41,192 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/177dc52f722342038fa3f6a76b6e33a9 is 50, key is test_row_0/C:col10/1731132999948/Put/seqid=0 2024-11-09T06:16:41,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742117_1293 (size=12301) 2024-11-09T06:16:41,206 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/ed017d7e412a4ecbb04e49ba6f158f58 2024-11-09T06:16:41,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742118_1294 (size=12797) 2024-11-09T06:16:41,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/69f5f40527424de681ed8a2ed6f7abe0 is 50, key is test_row_0/C:col10/1731133000630/Put/seqid=0 2024-11-09T06:16:41,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742119_1295 (size=12301) 2024-11-09T06:16:41,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:41,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133061298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:41,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133061299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133061300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:41,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133061412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133061412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133061412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T06:16:41,570 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/72ad73e04f3945028e0bfa39987d8915 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/72ad73e04f3945028e0bfa39987d8915 2024-11-09T06:16:41,581 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ddebd30416e9671cc930feb19be277/A of 96ddebd30416e9671cc930feb19be277 into 72ad73e04f3945028e0bfa39987d8915(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:41,581 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:41,581 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/A, priority=12, startTime=1731133001104; duration=0sec 2024-11-09T06:16:41,581 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:41,581 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:A 2024-11-09T06:16:41,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133061622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,626 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/177dc52f722342038fa3f6a76b6e33a9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/177dc52f722342038fa3f6a76b6e33a9 2024-11-09T06:16:41,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133061622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:41,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133061623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:41,632 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ddebd30416e9671cc930feb19be277/C of 96ddebd30416e9671cc930feb19be277 into 177dc52f722342038fa3f6a76b6e33a9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:41,633 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:41,633 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/C, priority=12, startTime=1731133001105; duration=0sec 2024-11-09T06:16:41,633 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:41,633 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:C 2024-11-09T06:16:41,662 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/69f5f40527424de681ed8a2ed6f7abe0 2024-11-09T06:16:41,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/0f2b9d70507c448aa4a3c5d2b351fab9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/0f2b9d70507c448aa4a3c5d2b351fab9 2024-11-09T06:16:41,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/0f2b9d70507c448aa4a3c5d2b351fab9, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T06:16:41,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/ed017d7e412a4ecbb04e49ba6f158f58 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/ed017d7e412a4ecbb04e49ba6f158f58 2024-11-09T06:16:41,682 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/ed017d7e412a4ecbb04e49ba6f158f58, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T06:16:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/69f5f40527424de681ed8a2ed6f7abe0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/69f5f40527424de681ed8a2ed6f7abe0 2024-11-09T06:16:41,687 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/69f5f40527424de681ed8a2ed6f7abe0, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T06:16:41,688 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 96ddebd30416e9671cc930feb19be277 in 558ms, sequenceid=274, compaction requested=false 2024-11-09T06:16:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-09T06:16:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-09T06:16:41,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-09T06:16:41,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 870 msec 2024-11-09T06:16:41,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 878 msec 2024-11-09T06:16:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:41,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:16:41,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:41,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:41,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:41,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:41,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:41,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:41,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/a8157a2aa2e04f3d85117c35e9c19fe4 is 50, key is test_row_0/A:col10/1731133001297/Put/seqid=0 2024-11-09T06:16:41,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742120_1296 (size=12301) 2024-11-09T06:16:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T06:16:41,953 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:41,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133061949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,954 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-09T06:16:41,956 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T06:16:41,956 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:41,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:41,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133061955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133061955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T06:16:42,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133062063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133062064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133062064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T06:16:42,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:42,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T06:16:42,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:42,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T06:16:42,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133062272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133062273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133062273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/a8157a2aa2e04f3d85117c35e9c19fe4 2024-11-09T06:16:42,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/a483cbade45c43aa9c17f768d54d7060 is 50, key is test_row_0/B:col10/1731133001297/Put/seqid=0 2024-11-09T06:16:42,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742121_1297 (size=12301) 2024-11-09T06:16:42,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T06:16:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T06:16:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T06:16:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133062583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133062583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133062583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T06:16:42,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:42,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/a483cbade45c43aa9c17f768d54d7060 2024-11-09T06:16:42,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/bbd101819d26409abb314448bd2db5c6 is 50, key is test_row_0/C:col10/1731133001297/Put/seqid=0 2024-11-09T06:16:42,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742122_1298 (size=12301) 2024-11-09T06:16:42,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133062832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:42,834 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in 96ddebd30416e9671cc930feb19be277 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:16:42,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133062842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:42,846 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in 96ddebd30416e9671cc930feb19be277 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:16:42,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:42,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:42,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T06:16:42,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:42,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:42,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:43,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T06:16:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:43,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:43,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T06:16:43,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133063092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:43,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133063093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:43,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133063093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:43,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T06:16:43,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:43,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:43,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:43,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:43,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:43,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/bbd101819d26409abb314448bd2db5c6 2024-11-09T06:16:43,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/a8157a2aa2e04f3d85117c35e9c19fe4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/a8157a2aa2e04f3d85117c35e9c19fe4 2024-11-09T06:16:43,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/a8157a2aa2e04f3d85117c35e9c19fe4, entries=150, sequenceid=297, filesize=12.0 K 2024-11-09T06:16:43,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/a483cbade45c43aa9c17f768d54d7060 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a483cbade45c43aa9c17f768d54d7060 2024-11-09T06:16:43,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a483cbade45c43aa9c17f768d54d7060, entries=150, sequenceid=297, filesize=12.0 K 2024-11-09T06:16:43,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/bbd101819d26409abb314448bd2db5c6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/bbd101819d26409abb314448bd2db5c6 2024-11-09T06:16:43,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/bbd101819d26409abb314448bd2db5c6, entries=150, sequenceid=297, filesize=12.0 K 2024-11-09T06:16:43,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 96ddebd30416e9671cc930feb19be277 in 1287ms, sequenceid=297, compaction requested=true 2024-11-09T06:16:43,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:43,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:43,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:43,224 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:43,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:43,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:43,224 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:43,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:43,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:43,225 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:43,225 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/A is initiating minor compaction (all files) 2024-11-09T06:16:43,225 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/A in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:43,225 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/72ad73e04f3945028e0bfa39987d8915, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/0f2b9d70507c448aa4a3c5d2b351fab9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/a8157a2aa2e04f3d85117c35e9c19fe4] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=36.5 K 2024-11-09T06:16:43,226 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:43,226 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/B is initiating minor compaction (all files) 2024-11-09T06:16:43,226 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/B in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:43,226 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/82665f34d08f4cd5aa6c6d83d28aa35f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/ed017d7e412a4ecbb04e49ba6f158f58, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a483cbade45c43aa9c17f768d54d7060] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=36.5 K 2024-11-09T06:16:43,226 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72ad73e04f3945028e0bfa39987d8915, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731132999948 2024-11-09T06:16:43,227 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 82665f34d08f4cd5aa6c6d83d28aa35f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731132999948 2024-11-09T06:16:43,227 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f2b9d70507c448aa4a3c5d2b351fab9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731133000623 2024-11-09T06:16:43,227 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting ed017d7e412a4ecbb04e49ba6f158f58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731133000623 2024-11-09T06:16:43,227 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8157a2aa2e04f3d85117c35e9c19fe4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731133001290 2024-11-09T06:16:43,228 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a483cbade45c43aa9c17f768d54d7060, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731133001290 2024-11-09T06:16:43,236 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#A#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:43,236 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/6314a5f1e88f4595af0277190b6a55ce is 50, key is test_row_0/A:col10/1731133001297/Put/seqid=0 2024-11-09T06:16:43,241 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#B#compaction#252 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:43,242 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/78a9f6d0a4184ea890d6b63dc314407c is 50, key is test_row_0/B:col10/1731133001297/Put/seqid=0 2024-11-09T06:16:43,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742123_1299 (size=12949) 2024-11-09T06:16:43,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742124_1300 (size=12949) 2024-11-09T06:16:43,274 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/78a9f6d0a4184ea890d6b63dc314407c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/78a9f6d0a4184ea890d6b63dc314407c 2024-11-09T06:16:43,285 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/B of 96ddebd30416e9671cc930feb19be277 into 78a9f6d0a4184ea890d6b63dc314407c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:43,286 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:43,286 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/B, priority=13, startTime=1731133003224; duration=0sec 2024-11-09T06:16:43,286 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:43,286 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:B 2024-11-09T06:16:43,286 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:43,288 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:43,288 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/C is initiating minor compaction (all files) 2024-11-09T06:16:43,288 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/C in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:43,289 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/177dc52f722342038fa3f6a76b6e33a9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/69f5f40527424de681ed8a2ed6f7abe0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/bbd101819d26409abb314448bd2db5c6] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=36.5 K 2024-11-09T06:16:43,289 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 177dc52f722342038fa3f6a76b6e33a9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731132999948 2024-11-09T06:16:43,290 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 69f5f40527424de681ed8a2ed6f7abe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731133000623 2024-11-09T06:16:43,290 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting bbd101819d26409abb314448bd2db5c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731133001290 2024-11-09T06:16:43,307 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#C#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:43,307 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/b483032f198e4be69861c0fcc0999688 is 50, key is test_row_0/C:col10/1731133001297/Put/seqid=0 2024-11-09T06:16:43,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742125_1301 (size=12949) 2024-11-09T06:16:43,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T06:16:43,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:43,336 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:16:43,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:43,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:43,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:43,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:43,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:43,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:43,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/963928a9997c42e780207feec0c3bb69 is 50, key is test_row_0/A:col10/1731133001953/Put/seqid=0 2024-11-09T06:16:43,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742126_1302 (size=12301) 2024-11-09T06:16:43,352 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/963928a9997c42e780207feec0c3bb69 2024-11-09T06:16:43,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/44332ed3ab974a9783361707d1710003 is 50, key is test_row_0/B:col10/1731133001953/Put/seqid=0 2024-11-09T06:16:43,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742127_1303 (size=12301) 2024-11-09T06:16:43,367 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/44332ed3ab974a9783361707d1710003 2024-11-09T06:16:43,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/0d5d0d61e06845cd860c51bca3460589 is 50, key is test_row_0/C:col10/1731133001953/Put/seqid=0 2024-11-09T06:16:43,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742128_1304 (size=12301) 2024-11-09T06:16:43,669 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/6314a5f1e88f4595af0277190b6a55ce as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6314a5f1e88f4595af0277190b6a55ce 2024-11-09T06:16:43,677 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/A of 96ddebd30416e9671cc930feb19be277 into 6314a5f1e88f4595af0277190b6a55ce(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:43,677 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:43,677 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/A, priority=13, startTime=1731133003224; duration=0sec 2024-11-09T06:16:43,677 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:43,677 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:A 2024-11-09T06:16:43,723 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/b483032f198e4be69861c0fcc0999688 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/b483032f198e4be69861c0fcc0999688 2024-11-09T06:16:43,738 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/C of 96ddebd30416e9671cc930feb19be277 into b483032f198e4be69861c0fcc0999688(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:43,738 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:43,738 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/C, priority=13, startTime=1731133003224; duration=0sec 2024-11-09T06:16:43,738 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:43,738 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:C 2024-11-09T06:16:43,800 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/0d5d0d61e06845cd860c51bca3460589 2024-11-09T06:16:43,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/963928a9997c42e780207feec0c3bb69 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/963928a9997c42e780207feec0c3bb69 2024-11-09T06:16:43,828 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/963928a9997c42e780207feec0c3bb69, entries=150, sequenceid=311, filesize=12.0 K 2024-11-09T06:16:43,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/44332ed3ab974a9783361707d1710003 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/44332ed3ab974a9783361707d1710003 2024-11-09T06:16:43,841 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/44332ed3ab974a9783361707d1710003, entries=150, sequenceid=311, filesize=12.0 K 2024-11-09T06:16:43,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/0d5d0d61e06845cd860c51bca3460589 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0d5d0d61e06845cd860c51bca3460589 2024-11-09T06:16:43,849 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0d5d0d61e06845cd860c51bca3460589, entries=150, sequenceid=311, filesize=12.0 K 2024-11-09T06:16:43,850 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 96ddebd30416e9671cc930feb19be277 in 514ms, sequenceid=311, compaction requested=false 2024-11-09T06:16:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-09T06:16:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-09T06:16:43,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-09T06:16:43,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8950 sec 2024-11-09T06:16:43,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.9010 sec 2024-11-09T06:16:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T06:16:44,103 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:44,105 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-09T06:16:44,107 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T06:16:44,109 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:44,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:44,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:16:44,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:44,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:44,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:44,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:44,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:44,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:44,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f9bd692ee2664a29b8884a41725e94b2 is 50, key is test_row_0/A:col10/1731133004107/Put/seqid=0 2024-11-09T06:16:44,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742129_1305 (size=12297) 2024-11-09T06:16:44,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133004155, max timestamp=1731133004162Num uniques 859; 2024-11-09T06:16:44,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133004155, max timestamp=1731133004162Num uniques 859; 2024-11-09T06:16:44,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:44,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:44,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133004155, max timestamp=1731133004164Num uniques 859; 2024-11-09T06:16:44,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:44,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133064197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133064198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133064198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T06:16:44,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T06:16:44,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:44,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:44,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:44,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133064312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133064313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133064313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T06:16:44,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T06:16:44,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:44,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:44,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:44,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133064523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133064523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133064523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f9bd692ee2664a29b8884a41725e94b2 2024-11-09T06:16:44,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/224e75aa7b5e493fb4c683b94e4c93bd is 50, key is test_row_0/B:col10/1731133004107/Put/seqid=0 2024-11-09T06:16:44,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742130_1306 (size=9857) 2024-11-09T06:16:44,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T06:16:44,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:44,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:44,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:44,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T06:16:44,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T06:16:44,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:44,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:44,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:44,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133064832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133064832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133064832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:44,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T06:16:44,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:44,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:44,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:44,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:44,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/224e75aa7b5e493fb4c683b94e4c93bd 2024-11-09T06:16:44,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/4c1df0142ba248f0997626d3367ed15c is 50, key is test_row_0/C:col10/1731133004107/Put/seqid=0 2024-11-09T06:16:44,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742131_1307 (size=9857) 2024-11-09T06:16:45,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T06:16:45,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:45,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:45,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:45,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:45,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:45,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T06:16:45,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:45,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:45,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:45,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:45,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T06:16:45,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133065342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:45,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133065342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:45,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T06:16:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:45,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:45,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:45,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133065353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:45,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/4c1df0142ba248f0997626d3367ed15c 2024-11-09T06:16:45,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f9bd692ee2664a29b8884a41725e94b2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f9bd692ee2664a29b8884a41725e94b2 2024-11-09T06:16:45,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f9bd692ee2664a29b8884a41725e94b2, entries=150, sequenceid=324, filesize=12.0 K 2024-11-09T06:16:45,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/224e75aa7b5e493fb4c683b94e4c93bd as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/224e75aa7b5e493fb4c683b94e4c93bd 2024-11-09T06:16:45,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/224e75aa7b5e493fb4c683b94e4c93bd, entries=100, sequenceid=324, filesize=9.6 K 2024-11-09T06:16:45,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/4c1df0142ba248f0997626d3367ed15c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/4c1df0142ba248f0997626d3367ed15c 2024-11-09T06:16:45,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/4c1df0142ba248f0997626d3367ed15c, entries=100, sequenceid=324, filesize=9.6 K 2024-11-09T06:16:45,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 96ddebd30416e9671cc930feb19be277 in 1295ms, sequenceid=324, compaction requested=true 2024-11-09T06:16:45,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:45,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:45,409 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:45,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:45,409 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:45,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:45,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:45,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:45,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:45,411 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:45,411 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37547 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:45,411 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/B is initiating minor compaction (all files) 2024-11-09T06:16:45,411 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/B in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:45,411 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/A is initiating minor compaction (all files) 2024-11-09T06:16:45,411 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/78a9f6d0a4184ea890d6b63dc314407c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/44332ed3ab974a9783361707d1710003, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/224e75aa7b5e493fb4c683b94e4c93bd] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=34.3 K 2024-11-09T06:16:45,411 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/A in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:45,411 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6314a5f1e88f4595af0277190b6a55ce, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/963928a9997c42e780207feec0c3bb69, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f9bd692ee2664a29b8884a41725e94b2] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=36.7 K 2024-11-09T06:16:45,412 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 78a9f6d0a4184ea890d6b63dc314407c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731133001290 2024-11-09T06:16:45,412 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6314a5f1e88f4595af0277190b6a55ce, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731133001290 2024-11-09T06:16:45,412 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 44332ed3ab974a9783361707d1710003, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731133001946 2024-11-09T06:16:45,413 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 963928a9997c42e780207feec0c3bb69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731133001946 2024-11-09T06:16:45,413 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9bd692ee2664a29b8884a41725e94b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731133004107 2024-11-09T06:16:45,413 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 224e75aa7b5e493fb4c683b94e4c93bd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731133004107 2024-11-09T06:16:45,427 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#A#compaction#260 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:45,427 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:45,428 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/9af30a7616c148448568f537d97ea50c is 50, key is test_row_0/B:col10/1731133004107/Put/seqid=0 2024-11-09T06:16:45,428 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/02a5e40873de451ab8f50e6bf943465c is 50, key is test_row_0/A:col10/1731133004107/Put/seqid=0 2024-11-09T06:16:45,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742132_1308 (size=13051) 2024-11-09T06:16:45,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T06:16:45,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:45,508 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-09T06:16:45,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:45,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:45,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/6b14eb091f1b421abe9684ddbe73ea61 is 50, key is test_row_0/A:col10/1731133004196/Put/seqid=0 2024-11-09T06:16:45,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742134_1310 (size=12301) 2024-11-09T06:16:45,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742133_1309 (size=13051) 2024-11-09T06:16:45,866 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/02a5e40873de451ab8f50e6bf943465c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/02a5e40873de451ab8f50e6bf943465c 2024-11-09T06:16:45,871 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/A of 96ddebd30416e9671cc930feb19be277 into 02a5e40873de451ab8f50e6bf943465c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:45,871 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:45,871 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/A, priority=13, startTime=1731133005409; duration=0sec 2024-11-09T06:16:45,871 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:45,871 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:A 2024-11-09T06:16:45,871 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:45,873 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:45,873 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/C is initiating minor compaction (all files) 2024-11-09T06:16:45,873 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/C in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:45,873 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/b483032f198e4be69861c0fcc0999688, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0d5d0d61e06845cd860c51bca3460589, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/4c1df0142ba248f0997626d3367ed15c] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=34.3 K 2024-11-09T06:16:45,873 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b483032f198e4be69861c0fcc0999688, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731133001290 2024-11-09T06:16:45,874 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d5d0d61e06845cd860c51bca3460589, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731133001946 2024-11-09T06:16:45,875 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c1df0142ba248f0997626d3367ed15c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731133004107 2024-11-09T06:16:45,883 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#C#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:45,883 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/a9a042d555f44946bc3442df38aaf665 is 50, key is test_row_0/C:col10/1731133004107/Put/seqid=0 2024-11-09T06:16:45,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742135_1311 (size=13051) 2024-11-09T06:16:45,895 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/a9a042d555f44946bc3442df38aaf665 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/a9a042d555f44946bc3442df38aaf665 2024-11-09T06:16:45,899 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/C of 96ddebd30416e9671cc930feb19be277 into a9a042d555f44946bc3442df38aaf665(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:45,899 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:45,899 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/C, priority=13, startTime=1731133005409; duration=0sec 2024-11-09T06:16:45,899 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:45,899 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:C 2024-11-09T06:16:45,943 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/6b14eb091f1b421abe9684ddbe73ea61 2024-11-09T06:16:45,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/30a971d556d54bb99ee1badaf218fbef is 50, key is test_row_0/B:col10/1731133004196/Put/seqid=0 2024-11-09T06:16:45,973 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/9af30a7616c148448568f537d97ea50c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/9af30a7616c148448568f537d97ea50c 2024-11-09T06:16:45,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742136_1312 (size=12301) 2024-11-09T06:16:45,985 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/B of 96ddebd30416e9671cc930feb19be277 into 9af30a7616c148448568f537d97ea50c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:45,985 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:45,985 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/B, priority=13, startTime=1731133005409; duration=0sec 2024-11-09T06:16:45,986 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:45,986 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:B 2024-11-09T06:16:45,990 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/30a971d556d54bb99ee1badaf218fbef 2024-11-09T06:16:46,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/654e38b7998945188e9bc9ba86796938 is 50, key is test_row_0/C:col10/1731133004196/Put/seqid=0 2024-11-09T06:16:46,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742137_1313 (size=12301) 2024-11-09T06:16:46,027 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/654e38b7998945188e9bc9ba86796938 2024-11-09T06:16:46,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/6b14eb091f1b421abe9684ddbe73ea61 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6b14eb091f1b421abe9684ddbe73ea61 2024-11-09T06:16:46,041 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6b14eb091f1b421abe9684ddbe73ea61, entries=150, sequenceid=360, filesize=12.0 K 2024-11-09T06:16:46,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/30a971d556d54bb99ee1badaf218fbef as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/30a971d556d54bb99ee1badaf218fbef 2024-11-09T06:16:46,049 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/30a971d556d54bb99ee1badaf218fbef, entries=150, sequenceid=360, filesize=12.0 K 2024-11-09T06:16:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/654e38b7998945188e9bc9ba86796938 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/654e38b7998945188e9bc9ba86796938 2024-11-09T06:16:46,059 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/654e38b7998945188e9bc9ba86796938, entries=150, sequenceid=360, filesize=12.0 K 2024-11-09T06:16:46,060 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=0 B/0 for 96ddebd30416e9671cc930feb19be277 in 552ms, sequenceid=360, compaction requested=false 2024-11-09T06:16:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-09T06:16:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-09T06:16:46,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-09T06:16:46,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9530 sec 2024-11-09T06:16:46,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9610 sec 2024-11-09T06:16:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T06:16:46,255 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:46,256 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-09T06:16:46,258 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T06:16:46,259 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:46,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T06:16:46,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:16:46,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:46,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:46,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:46,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:46,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:46,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:46,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/4160a9cf223e46d2ad1fc2d4da07bf81 is 50, key is test_row_0/A:col10/1731133006368/Put/seqid=0 2024-11-09T06:16:46,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742138_1314 (size=12297) 2024-11-09T06:16:46,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/4160a9cf223e46d2ad1fc2d4da07bf81 2024-11-09T06:16:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731133006404, max timestamp=1731133006414Num uniques 859; 2024-11-09T06:16:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133006404, max timestamp=1731133006414Num uniques 859; 2024-11-09T06:16:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731133006404, max timestamp=1731133006414Num uniques 859; 2024-11-09T06:16:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:16:46,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/b175926f925c4eda91548a05853666ec is 50, key is test_row_0/B:col10/1731133006368/Put/seqid=0 2024-11-09T06:16:46,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T06:16:46,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:46,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:46,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:46,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742139_1315 (size=9857) 2024-11-09T06:16:46,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/b175926f925c4eda91548a05853666ec 2024-11-09T06:16:46,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133066460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:46,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133066463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:46,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133066464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:46,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:46,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/e5f6e1ac1ba543b2a24301a4f0a3b866 is 50, key is test_row_0/C:col10/1731133006368/Put/seqid=0 2024-11-09T06:16:46,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742140_1316 (size=9857) 2024-11-09T06:16:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T06:16:46,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133066573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:46,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133066583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133066583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:46,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T06:16:46,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:46,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:46,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:46,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T06:16:46,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:46,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:46,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:46,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133066783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:46,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:46,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133066792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:46,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133066792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:46,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47328 deadline: 1731133066853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:46,854 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in 96ddebd30416e9671cc930feb19be277 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:16:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:46,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47338 deadline: 1731133066872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:46,876 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in 96ddebd30416e9671cc930feb19be277 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:16:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T06:16:46,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T06:16:46,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:46,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:46,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:46,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:46,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/e5f6e1ac1ba543b2a24301a4f0a3b866 2024-11-09T06:16:46,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/4160a9cf223e46d2ad1fc2d4da07bf81 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/4160a9cf223e46d2ad1fc2d4da07bf81 2024-11-09T06:16:46,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/4160a9cf223e46d2ad1fc2d4da07bf81, entries=150, sequenceid=373, filesize=12.0 K 2024-11-09T06:16:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/b175926f925c4eda91548a05853666ec as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/b175926f925c4eda91548a05853666ec 2024-11-09T06:16:46,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/b175926f925c4eda91548a05853666ec, entries=100, sequenceid=373, filesize=9.6 K 2024-11-09T06:16:46,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/e5f6e1ac1ba543b2a24301a4f0a3b866 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/e5f6e1ac1ba543b2a24301a4f0a3b866 2024-11-09T06:16:46,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/e5f6e1ac1ba543b2a24301a4f0a3b866, entries=100, sequenceid=373, filesize=9.6 K 2024-11-09T06:16:46,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 96ddebd30416e9671cc930feb19be277 in 582ms, sequenceid=373, compaction requested=true 2024-11-09T06:16:46,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:46,955 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:46,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:46,957 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:46,957 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/A is initiating minor compaction (all files) 2024-11-09T06:16:46,957 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/A in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:46,957 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/02a5e40873de451ab8f50e6bf943465c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6b14eb091f1b421abe9684ddbe73ea61, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/4160a9cf223e46d2ad1fc2d4da07bf81] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=36.8 K 2024-11-09T06:16:46,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:46,957 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:46,958 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02a5e40873de451ab8f50e6bf943465c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731133001947 2024-11-09T06:16:46,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:46,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:46,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:46,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:46,959 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:46,959 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b14eb091f1b421abe9684ddbe73ea61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731133004185 2024-11-09T06:16:46,959 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/B is initiating minor compaction (all files) 2024-11-09T06:16:46,959 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/B in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:46,959 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/9af30a7616c148448568f537d97ea50c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/30a971d556d54bb99ee1badaf218fbef, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/b175926f925c4eda91548a05853666ec] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=34.4 K 2024-11-09T06:16:46,960 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4160a9cf223e46d2ad1fc2d4da07bf81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731133006368 2024-11-09T06:16:46,960 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af30a7616c148448568f537d97ea50c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731133001947 2024-11-09T06:16:46,960 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 30a971d556d54bb99ee1badaf218fbef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731133004185 2024-11-09T06:16:46,961 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b175926f925c4eda91548a05853666ec, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731133006368 2024-11-09T06:16:46,971 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#A#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:46,971 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/ba5082247f3045b9908c8529fee7e050 is 50, key is test_row_0/A:col10/1731133006368/Put/seqid=0 2024-11-09T06:16:46,989 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#B#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:46,989 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/ffbc3a8dc39542599f3d251409686052 is 50, key is test_row_0/B:col10/1731133006368/Put/seqid=0 2024-11-09T06:16:47,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742141_1317 (size=13153) 2024-11-09T06:16:47,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742142_1318 (size=13153) 2024-11-09T06:16:47,013 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/ba5082247f3045b9908c8529fee7e050 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/ba5082247f3045b9908c8529fee7e050 2024-11-09T06:16:47,021 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/A of 96ddebd30416e9671cc930feb19be277 into ba5082247f3045b9908c8529fee7e050(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:47,021 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:47,021 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/A, priority=13, startTime=1731133006955; duration=0sec 2024-11-09T06:16:47,021 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:47,021 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:A 2024-11-09T06:16:47,021 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:47,023 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:47,023 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/C is initiating minor compaction (all files) 2024-11-09T06:16:47,023 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/C in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:47,023 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/a9a042d555f44946bc3442df38aaf665, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/654e38b7998945188e9bc9ba86796938, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/e5f6e1ac1ba543b2a24301a4f0a3b866] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=34.4 K 2024-11-09T06:16:47,023 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9a042d555f44946bc3442df38aaf665, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731133001947 2024-11-09T06:16:47,024 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 654e38b7998945188e9bc9ba86796938, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731133004185 2024-11-09T06:16:47,024 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5f6e1ac1ba543b2a24301a4f0a3b866, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731133006368 2024-11-09T06:16:47,032 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#C#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:47,032 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/8dd15ebe84e9470b957eb1b6c7d4539d is 50, key is test_row_0/C:col10/1731133006368/Put/seqid=0 2024-11-09T06:16:47,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T06:16:47,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:47,048 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-09T06:16:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:47,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/e53972cd2545450ab0faaec60390b54f is 50, key is test_row_0/A:col10/1731133006457/Put/seqid=0 2024-11-09T06:16:47,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742143_1319 (size=13153) 2024-11-09T06:16:47,090 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/8dd15ebe84e9470b957eb1b6c7d4539d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/8dd15ebe84e9470b957eb1b6c7d4539d 2024-11-09T06:16:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:47,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:47,100 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/C of 96ddebd30416e9671cc930feb19be277 into 8dd15ebe84e9470b957eb1b6c7d4539d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:47,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742144_1320 (size=12301) 2024-11-09T06:16:47,100 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:47,100 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/C, priority=13, startTime=1731133006958; duration=0sec 2024-11-09T06:16:47,100 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:47,101 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:C 2024-11-09T06:16:47,102 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/e53972cd2545450ab0faaec60390b54f 2024-11-09T06:16:47,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/be3c8d74137040d096959c12a5a0c501 is 50, key is test_row_0/B:col10/1731133006457/Put/seqid=0 2024-11-09T06:16:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133067122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133067122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133067123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742145_1321 (size=12301) 2024-11-09T06:16:47,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133067232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133067232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133067243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T06:16:47,416 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/ffbc3a8dc39542599f3d251409686052 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/ffbc3a8dc39542599f3d251409686052 2024-11-09T06:16:47,425 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/B of 96ddebd30416e9671cc930feb19be277 into ffbc3a8dc39542599f3d251409686052(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:47,426 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:47,426 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/B, priority=13, startTime=1731133006957; duration=0sec 2024-11-09T06:16:47,426 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:47,426 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:B 2024-11-09T06:16:47,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133067442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133067443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133067453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,558 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/be3c8d74137040d096959c12a5a0c501 2024-11-09T06:16:47,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/9c9461fb67fc4410953102a96992f4e7 is 50, key is test_row_0/C:col10/1731133006457/Put/seqid=0 2024-11-09T06:16:47,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742146_1322 (size=12301) 2024-11-09T06:16:47,581 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/9c9461fb67fc4410953102a96992f4e7 2024-11-09T06:16:47,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/e53972cd2545450ab0faaec60390b54f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/e53972cd2545450ab0faaec60390b54f 2024-11-09T06:16:47,597 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/e53972cd2545450ab0faaec60390b54f, entries=150, sequenceid=410, filesize=12.0 K 2024-11-09T06:16:47,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/be3c8d74137040d096959c12a5a0c501 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/be3c8d74137040d096959c12a5a0c501 2024-11-09T06:16:47,608 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/be3c8d74137040d096959c12a5a0c501, entries=150, sequenceid=410, filesize=12.0 K 2024-11-09T06:16:47,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/9c9461fb67fc4410953102a96992f4e7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9c9461fb67fc4410953102a96992f4e7 2024-11-09T06:16:47,626 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9c9461fb67fc4410953102a96992f4e7, entries=150, sequenceid=410, filesize=12.0 K 2024-11-09T06:16:47,627 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=53.67 KB/54960 for 96ddebd30416e9671cc930feb19be277 in 580ms, sequenceid=410, compaction requested=false 2024-11-09T06:16:47,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:47,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:47,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-09T06:16:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-09T06:16:47,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-09T06:16:47,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3690 sec 2024-11-09T06:16:47,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.3750 sec 2024-11-09T06:16:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:47,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T06:16:47,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:47,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:47,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:47,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:47,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:47,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:47,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/be9bf9a4b1a54433880be5604efd6e8c is 50, key is test_row_0/A:col10/1731133007119/Put/seqid=0 2024-11-09T06:16:47,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742147_1323 (size=14741) 2024-11-09T06:16:47,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133067787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133067787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133067788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133067903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133067903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133067903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133068113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133068113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133068113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/be9bf9a4b1a54433880be5604efd6e8c 2024-11-09T06:16:48,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/c4b7eb1d41ae40f999b7c0f2f5b0e233 is 50, key is test_row_0/B:col10/1731133007119/Put/seqid=0 2024-11-09T06:16:48,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742148_1324 (size=12301) 2024-11-09T06:16:48,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/c4b7eb1d41ae40f999b7c0f2f5b0e233 2024-11-09T06:16:48,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/f5217709e01146c4855440e8844b7ea0 is 50, key is test_row_0/C:col10/1731133007119/Put/seqid=0 2024-11-09T06:16:48,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742149_1325 (size=12301) 2024-11-09T06:16:48,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/f5217709e01146c4855440e8844b7ea0 2024-11-09T06:16:48,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/be9bf9a4b1a54433880be5604efd6e8c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/be9bf9a4b1a54433880be5604efd6e8c 2024-11-09T06:16:48,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/be9bf9a4b1a54433880be5604efd6e8c, entries=200, sequenceid=425, filesize=14.4 K 2024-11-09T06:16:48,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/c4b7eb1d41ae40f999b7c0f2f5b0e233 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/c4b7eb1d41ae40f999b7c0f2f5b0e233 2024-11-09T06:16:48,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/c4b7eb1d41ae40f999b7c0f2f5b0e233, entries=150, sequenceid=425, filesize=12.0 K 2024-11-09T06:16:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/f5217709e01146c4855440e8844b7ea0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/f5217709e01146c4855440e8844b7ea0 2024-11-09T06:16:48,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/f5217709e01146c4855440e8844b7ea0, entries=150, sequenceid=425, filesize=12.0 K 2024-11-09T06:16:48,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 96ddebd30416e9671cc930feb19be277 in 494ms, sequenceid=425, compaction requested=true 2024-11-09T06:16:48,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:48,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:48,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:48,250 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:48,250 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:48,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:48,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:48,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:48,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:48,252 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:48,252 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/B is initiating minor compaction (all files) 2024-11-09T06:16:48,252 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:48,252 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/B in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:48,252 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/A is initiating minor compaction (all files) 2024-11-09T06:16:48,252 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/A in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:48,252 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/ffbc3a8dc39542599f3d251409686052, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/be3c8d74137040d096959c12a5a0c501, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/c4b7eb1d41ae40f999b7c0f2f5b0e233] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=36.9 K 2024-11-09T06:16:48,252 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/ba5082247f3045b9908c8529fee7e050, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/e53972cd2545450ab0faaec60390b54f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/be9bf9a4b1a54433880be5604efd6e8c] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=39.3 K 2024-11-09T06:16:48,253 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting ffbc3a8dc39542599f3d251409686052, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731133004197 2024-11-09T06:16:48,253 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba5082247f3045b9908c8529fee7e050, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731133004197 2024-11-09T06:16:48,253 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting be3c8d74137040d096959c12a5a0c501, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731133006457 2024-11-09T06:16:48,253 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e53972cd2545450ab0faaec60390b54f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731133006457 2024-11-09T06:16:48,254 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting c4b7eb1d41ae40f999b7c0f2f5b0e233, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731133007119 2024-11-09T06:16:48,254 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting be9bf9a4b1a54433880be5604efd6e8c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731133007119 2024-11-09T06:16:48,278 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#A#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:48,279 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/e90388a2f66b48a982b1d4a5868b91a7 is 50, key is test_row_0/A:col10/1731133007119/Put/seqid=0 2024-11-09T06:16:48,281 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:48,282 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/73b58b40e1a54da1bc17382104e096c4 is 50, key is test_row_0/B:col10/1731133007119/Put/seqid=0 2024-11-09T06:16:48,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742150_1326 (size=13255) 2024-11-09T06:16:48,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742151_1327 (size=13255) 2024-11-09T06:16:48,330 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/e90388a2f66b48a982b1d4a5868b91a7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/e90388a2f66b48a982b1d4a5868b91a7 2024-11-09T06:16:48,340 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/A of 96ddebd30416e9671cc930feb19be277 into e90388a2f66b48a982b1d4a5868b91a7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:48,340 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:48,340 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/A, priority=13, startTime=1731133008250; duration=0sec 2024-11-09T06:16:48,340 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:48,340 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:A 2024-11-09T06:16:48,340 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:48,341 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:48,341 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/C is initiating minor compaction (all files) 2024-11-09T06:16:48,342 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/C in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:48,342 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/8dd15ebe84e9470b957eb1b6c7d4539d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9c9461fb67fc4410953102a96992f4e7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/f5217709e01146c4855440e8844b7ea0] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=36.9 K 2024-11-09T06:16:48,342 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dd15ebe84e9470b957eb1b6c7d4539d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731133004197 2024-11-09T06:16:48,343 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c9461fb67fc4410953102a96992f4e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731133006457 2024-11-09T06:16:48,343 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5217709e01146c4855440e8844b7ea0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731133007119 2024-11-09T06:16:48,353 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#C#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:48,353 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/1b68a420dd4a47b2861d80ea80c83906 is 50, key is test_row_0/C:col10/1731133007119/Put/seqid=0 2024-11-09T06:16:48,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742152_1328 (size=13255) 2024-11-09T06:16:48,364 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/1b68a420dd4a47b2861d80ea80c83906 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/1b68a420dd4a47b2861d80ea80c83906 2024-11-09T06:16:48,384 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/C of 96ddebd30416e9671cc930feb19be277 into 1b68a420dd4a47b2861d80ea80c83906(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:48,384 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:48,384 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/C, priority=13, startTime=1731133008250; duration=0sec 2024-11-09T06:16:48,384 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:48,384 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:C 2024-11-09T06:16:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T06:16:48,403 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:48,404 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:16:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-09T06:16:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T06:16:48,407 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:16:48,408 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:16:48,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:16:48,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-09T06:16:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:48,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:48,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:48,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:48,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:48,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:48,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:48,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/66c248edff934bf38181c0e833233e7e is 50, key is test_row_0/A:col10/1731133007785/Put/seqid=0 2024-11-09T06:16:48,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133068436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133068442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133068443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742153_1329 (size=14741) 2024-11-09T06:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T06:16:48,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133068552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133068552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133068553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T06:16:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:48,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:48,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T06:16:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:48,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T06:16:48,731 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/73b58b40e1a54da1bc17382104e096c4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/73b58b40e1a54da1bc17382104e096c4 2024-11-09T06:16:48,745 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/B of 96ddebd30416e9671cc930feb19be277 into 73b58b40e1a54da1bc17382104e096c4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:48,745 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:48,745 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/B, priority=13, startTime=1731133008250; duration=0sec 2024-11-09T06:16:48,745 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:48,745 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:B 2024-11-09T06:16:48,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133068763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133068763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133068763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:48,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/66c248edff934bf38181c0e833233e7e 2024-11-09T06:16:48,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/6214fe533d0c4168a1f10d488d370a14 is 50, key is test_row_0/B:col10/1731133007785/Put/seqid=0 2024-11-09T06:16:48,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T06:16:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:48,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:48,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742154_1330 (size=12301) 2024-11-09T06:16:48,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/6214fe533d0c4168a1f10d488d370a14 2024-11-09T06:16:48,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/9466d2971bba4c4ea26eb39ba18fcbaf is 50, key is test_row_0/C:col10/1731133007785/Put/seqid=0 2024-11-09T06:16:48,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742155_1331 (size=12301) 2024-11-09T06:16:49,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T06:16:49,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:49,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:49,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:49,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:49,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T06:16:49,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133069073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133069073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133069073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:49,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T06:16:49,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:49,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:49,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:49,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:49,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:49,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:16:49,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/9466d2971bba4c4ea26eb39ba18fcbaf 2024-11-09T06:16:49,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/66c248edff934bf38181c0e833233e7e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/66c248edff934bf38181c0e833233e7e 2024-11-09T06:16:49,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/66c248edff934bf38181c0e833233e7e, entries=200, sequenceid=449, filesize=14.4 K 2024-11-09T06:16:49,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/6214fe533d0c4168a1f10d488d370a14 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/6214fe533d0c4168a1f10d488d370a14 2024-11-09T06:16:49,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/6214fe533d0c4168a1f10d488d370a14, entries=150, sequenceid=449, filesize=12.0 K 2024-11-09T06:16:49,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/9466d2971bba4c4ea26eb39ba18fcbaf as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9466d2971bba4c4ea26eb39ba18fcbaf 2024-11-09T06:16:49,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9466d2971bba4c4ea26eb39ba18fcbaf, entries=150, sequenceid=449, filesize=12.0 K 2024-11-09T06:16:49,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 96ddebd30416e9671cc930feb19be277 in 883ms, sequenceid=449, compaction requested=false 2024-11-09T06:16:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:49,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T06:16:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:49,327 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:16:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:49,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:49,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:49,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:49,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:49,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/d6d06ca4c71c441da8f842238cdb633e is 50, key is test_row_0/A:col10/1731133008440/Put/seqid=0 2024-11-09T06:16:49,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742156_1332 (size=12301) 2024-11-09T06:16:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T06:16:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:49,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. as already flushing 2024-11-09T06:16:49,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133009605, max timestamp=1731133009615Num uniques 859; 2024-11-09T06:16:49,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133009605, max timestamp=1731133009615Num uniques 859; 2024-11-09T06:16:49,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:16:49,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:16:49,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133009605, max timestamp=1731133009615Num uniques 859; 2024-11-09T06:16:49,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:16:49,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133069652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:49,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133069655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133069657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:49,743 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/d6d06ca4c71c441da8f842238cdb633e 2024-11-09T06:16:49,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/a971f4414e8040b3976d967a89711ebf is 50, key is test_row_0/B:col10/1731133008440/Put/seqid=0 2024-11-09T06:16:49,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133069762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:49,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133069773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133069774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:49,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742157_1333 (size=12301) 2024-11-09T06:16:49,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133069974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:49,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133069982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133069982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:50,183 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/a971f4414e8040b3976d967a89711ebf 2024-11-09T06:16:50,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/0de9730a24ea4f98a9ca3dd063002da4 is 50, key is test_row_0/C:col10/1731133008440/Put/seqid=0 2024-11-09T06:16:50,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742158_1334 (size=12301) 2024-11-09T06:16:50,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:50,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133070283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:50,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133070292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:50,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133070292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:50,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T06:16:50,643 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/0de9730a24ea4f98a9ca3dd063002da4 2024-11-09T06:16:50,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/d6d06ca4c71c441da8f842238cdb633e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/d6d06ca4c71c441da8f842238cdb633e 2024-11-09T06:16:50,665 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/d6d06ca4c71c441da8f842238cdb633e, entries=150, sequenceid=460, filesize=12.0 K 2024-11-09T06:16:50,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/a971f4414e8040b3976d967a89711ebf as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a971f4414e8040b3976d967a89711ebf 2024-11-09T06:16:50,673 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a971f4414e8040b3976d967a89711ebf, entries=150, sequenceid=460, filesize=12.0 K 2024-11-09T06:16:50,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/0de9730a24ea4f98a9ca3dd063002da4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0de9730a24ea4f98a9ca3dd063002da4 2024-11-09T06:16:50,682 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0de9730a24ea4f98a9ca3dd063002da4, entries=150, sequenceid=460, filesize=12.0 K 2024-11-09T06:16:50,684 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 96ddebd30416e9671cc930feb19be277 in 1356ms, sequenceid=460, compaction requested=true 2024-11-09T06:16:50,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:50,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:50,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-09T06:16:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-09T06:16:50,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-09T06:16:50,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2780 sec 2024-11-09T06:16:50,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.2880 sec 2024-11-09T06:16:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:50,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-09T06:16:50,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:50,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:50,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:50,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:50,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:50,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:50,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/5fcd1a4466b044ca88d97db2afce509e is 50, key is test_row_0/A:col10/1731133009646/Put/seqid=0 2024-11-09T06:16:50,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133070831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:50,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133070840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133070841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:50,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742159_1335 (size=14741) 2024-11-09T06:16:50,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/5fcd1a4466b044ca88d97db2afce509e 2024-11-09T06:16:50,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/7fe6c1295aaa45fd9f1f52e4ac4526ff is 50, key is test_row_0/B:col10/1731133009646/Put/seqid=0 2024-11-09T06:16:50,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742160_1336 (size=12301) 2024-11-09T06:16:50,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/7fe6c1295aaa45fd9f1f52e4ac4526ff 2024-11-09T06:16:50,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/2e15227bfc534c9a80dce557bd7eb221 is 50, key is test_row_0/C:col10/1731133009646/Put/seqid=0 2024-11-09T06:16:50,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133070954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:50,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133070955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:50,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133070955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:50,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:50,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:50,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:50,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742161_1337 (size=12301) 2024-11-09T06:16:50,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/2e15227bfc534c9a80dce557bd7eb221 2024-11-09T06:16:50,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/5fcd1a4466b044ca88d97db2afce509e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/5fcd1a4466b044ca88d97db2afce509e 2024-11-09T06:16:50,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/5fcd1a4466b044ca88d97db2afce509e, entries=200, sequenceid=497, filesize=14.4 K 2024-11-09T06:16:50,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/7fe6c1295aaa45fd9f1f52e4ac4526ff as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/7fe6c1295aaa45fd9f1f52e4ac4526ff 2024-11-09T06:16:50,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/7fe6c1295aaa45fd9f1f52e4ac4526ff, entries=150, sequenceid=497, filesize=12.0 K 2024-11-09T06:16:50,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/2e15227bfc534c9a80dce557bd7eb221 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2e15227bfc534c9a80dce557bd7eb221 2024-11-09T06:16:51,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2e15227bfc534c9a80dce557bd7eb221, entries=150, sequenceid=497, filesize=12.0 K 2024-11-09T06:16:51,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 96ddebd30416e9671cc930feb19be277 in 202ms, sequenceid=497, compaction requested=true 2024-11-09T06:16:51,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:51,003 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:16:51,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:51,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:51,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:51,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:51,004 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:16:51,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:51,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:51,009 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55038 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:16:51,009 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/A is initiating minor compaction (all files) 2024-11-09T06:16:51,009 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/A in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:51,010 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/e90388a2f66b48a982b1d4a5868b91a7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/66c248edff934bf38181c0e833233e7e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/d6d06ca4c71c441da8f842238cdb633e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/5fcd1a4466b044ca88d97db2afce509e] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=53.7 K 2024-11-09T06:16:51,010 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:16:51,010 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/B is initiating minor compaction (all files) 2024-11-09T06:16:51,010 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/B in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:51,010 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/73b58b40e1a54da1bc17382104e096c4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/6214fe533d0c4168a1f10d488d370a14, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a971f4414e8040b3976d967a89711ebf, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/7fe6c1295aaa45fd9f1f52e4ac4526ff] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=49.0 K 2024-11-09T06:16:51,010 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e90388a2f66b48a982b1d4a5868b91a7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731133007119 2024-11-09T06:16:51,014 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 73b58b40e1a54da1bc17382104e096c4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731133007119 2024-11-09T06:16:51,015 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66c248edff934bf38181c0e833233e7e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731133007780 2024-11-09T06:16:51,015 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6214fe533d0c4168a1f10d488d370a14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731133007785 2024-11-09T06:16:51,015 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6d06ca4c71c441da8f842238cdb633e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731133008435 2024-11-09T06:16:51,016 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a971f4414e8040b3976d967a89711ebf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731133008435 2024-11-09T06:16:51,016 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fcd1a4466b044ca88d97db2afce509e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731133009636 2024-11-09T06:16:51,016 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe6c1295aaa45fd9f1f52e4ac4526ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731133009646 2024-11-09T06:16:51,036 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#A#compaction#290 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:51,036 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f6b0395ee02e4044b05695ac6ffd5ff2 is 50, key is test_row_0/A:col10/1731133009646/Put/seqid=0 2024-11-09T06:16:51,043 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#B#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:51,046 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/b9446649f5bd4ca5b1d1b05304b911b0 is 50, key is test_row_0/B:col10/1731133009646/Put/seqid=0 2024-11-09T06:16:51,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742162_1338 (size=13391) 2024-11-09T06:16:51,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742163_1339 (size=13391) 2024-11-09T06:16:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:51,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T06:16:51,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:51,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:51,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:51,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:51,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:51,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:51,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f951d6cfe3034ec986fa8d3e3c6f0de2 is 50, key is test_row_0/A:col10/1731133010830/Put/seqid=0 2024-11-09T06:16:51,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742164_1340 (size=14741) 2024-11-09T06:16:51,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133071220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133011203, max timestamp=1731133011223Num uniques 859; 2024-11-09T06:16:51,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:51,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133011203, max timestamp=1731133011223Num uniques 859; 2024-11-09T06:16:51,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:51,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133011203, max timestamp=1731133011223Num uniques 859; 2024-11-09T06:16:51,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:16:51,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133071303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133071306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133071333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133071412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133071422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,496 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/b9446649f5bd4ca5b1d1b05304b911b0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/b9446649f5bd4ca5b1d1b05304b911b0 2024-11-09T06:16:51,499 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f6b0395ee02e4044b05695ac6ffd5ff2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f6b0395ee02e4044b05695ac6ffd5ff2 2024-11-09T06:16:51,504 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ddebd30416e9671cc930feb19be277/B of 96ddebd30416e9671cc930feb19be277 into b9446649f5bd4ca5b1d1b05304b911b0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:51,504 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:51,504 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/B, priority=12, startTime=1731133011004; duration=0sec 2024-11-09T06:16:51,504 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:51,505 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:B 2024-11-09T06:16:51,505 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:16:51,506 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:16:51,506 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/C is initiating minor compaction (all files) 2024-11-09T06:16:51,506 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/C in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:51,506 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/1b68a420dd4a47b2861d80ea80c83906, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9466d2971bba4c4ea26eb39ba18fcbaf, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0de9730a24ea4f98a9ca3dd063002da4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2e15227bfc534c9a80dce557bd7eb221] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=49.0 K 2024-11-09T06:16:51,507 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b68a420dd4a47b2861d80ea80c83906, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731133007119 2024-11-09T06:16:51,507 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9466d2971bba4c4ea26eb39ba18fcbaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731133007785 2024-11-09T06:16:51,507 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de9730a24ea4f98a9ca3dd063002da4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731133008435 2024-11-09T06:16:51,508 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e15227bfc534c9a80dce557bd7eb221, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731133009646 2024-11-09T06:16:51,526 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#C#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:51,526 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/317cca11f4474eb6bc158bcb980ba995 is 50, key is test_row_0/C:col10/1731133009646/Put/seqid=0 2024-11-09T06:16:51,531 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ddebd30416e9671cc930feb19be277/A of 96ddebd30416e9671cc930feb19be277 into f6b0395ee02e4044b05695ac6ffd5ff2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:51,531 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:51,532 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/A, priority=12, startTime=1731133011003; duration=0sec 2024-11-09T06:16:51,534 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:51,534 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:A 2024-11-09T06:16:51,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133071543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742165_1341 (size=13391) 2024-11-09T06:16:51,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f951d6cfe3034ec986fa8d3e3c6f0de2 2024-11-09T06:16:51,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133071623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/862f5c5eb92b4415ae35ecd433bdd1dd is 50, key is test_row_0/B:col10/1731133010830/Put/seqid=0 2024-11-09T06:16:51,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133071633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742166_1342 (size=12301) 2024-11-09T06:16:51,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133071863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133071932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133071942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:51,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:51,976 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/317cca11f4474eb6bc158bcb980ba995 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/317cca11f4474eb6bc158bcb980ba995 2024-11-09T06:16:51,984 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ddebd30416e9671cc930feb19be277/C of 96ddebd30416e9671cc930feb19be277 into 317cca11f4474eb6bc158bcb980ba995(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:51,984 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:51,984 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/C, priority=12, startTime=1731133011004; duration=0sec 2024-11-09T06:16:51,985 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:51,985 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:C 2024-11-09T06:16:52,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/862f5c5eb92b4415ae35ecd433bdd1dd 2024-11-09T06:16:52,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/2cada715f1a848db8c276f1e21af4ac7 is 50, key is test_row_0/C:col10/1731133010830/Put/seqid=0 2024-11-09T06:16:52,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742167_1343 (size=12301) 2024-11-09T06:16:52,205 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-09T06:16:52,205 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:52,205 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:52,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:52,205 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:52,207 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-11-09T06:16:52,207 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:52,208 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:52,208 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:52,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:52,209 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-11-09T06:16:52,209 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:52,209 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:52,209 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:52,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:52,210 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-11-09T06:16:52,210 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:52,210 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:52,210 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:52,210 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:52,213 INFO [Thread-1133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1133. 2024-11-09T06:16:52,213 DEBUG [Thread-1133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:52,213 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:52,213 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:52,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:52,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47314 deadline: 1731133072372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:52,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:52,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:52,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:52,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731133072442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:52,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:16:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47292 deadline: 1731133072453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:16:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:16:52,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/2cada715f1a848db8c276f1e21af4ac7 2024-11-09T06:16:52,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/f951d6cfe3034ec986fa8d3e3c6f0de2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f951d6cfe3034ec986fa8d3e3c6f0de2 2024-11-09T06:16:52,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f951d6cfe3034ec986fa8d3e3c6f0de2, entries=200, sequenceid=508, filesize=14.4 K 2024-11-09T06:16:52,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/862f5c5eb92b4415ae35ecd433bdd1dd as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/862f5c5eb92b4415ae35ecd433bdd1dd 2024-11-09T06:16:52,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/862f5c5eb92b4415ae35ecd433bdd1dd, entries=150, sequenceid=508, filesize=12.0 K 2024-11-09T06:16:52,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/2cada715f1a848db8c276f1e21af4ac7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2cada715f1a848db8c276f1e21af4ac7 2024-11-09T06:16:52,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2cada715f1a848db8c276f1e21af4ac7, entries=150, sequenceid=508, filesize=12.0 K 2024-11-09T06:16:52,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 96ddebd30416e9671cc930feb19be277 in 1406ms, sequenceid=508, compaction requested=false 2024-11-09T06:16:52,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T06:16:52,583 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:53,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-09T06:16:53,385 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-09T06:16:53,385 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:53,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:53,385 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:53,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:53,385 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:53,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:53,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:53,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:53,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:16:53,385 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:53,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/126c8eac7eb54b36bce55eeb06101e7f is 50, key is test_row_0/A:col10/1731133011302/Put/seqid=0 2024-11-09T06:16:53,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742168_1344 (size=12301) 2024-11-09T06:16:53,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/126c8eac7eb54b36bce55eeb06101e7f 2024-11-09T06:16:53,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/49a3d83d18b64f95ae2d32bea0494fee is 50, key is test_row_0/B:col10/1731133011302/Put/seqid=0 2024-11-09T06:16:53,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742169_1345 (size=12301) 2024-11-09T06:16:53,454 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-09T06:16:53,454 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:53,454 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:53,454 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:53,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:53,465 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-09T06:16:53,465 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:53,465 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:53,465 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:53,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:53,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/49a3d83d18b64f95ae2d32bea0494fee 2024-11-09T06:16:53,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/a2789bd420fd49ad99a3c78a2eb9980f is 50, key is test_row_0/C:col10/1731133011302/Put/seqid=0 2024-11-09T06:16:53,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742170_1346 (size=12301) 2024-11-09T06:16:53,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/a2789bd420fd49ad99a3c78a2eb9980f 2024-11-09T06:16:53,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/126c8eac7eb54b36bce55eeb06101e7f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/126c8eac7eb54b36bce55eeb06101e7f 2024-11-09T06:16:53,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/126c8eac7eb54b36bce55eeb06101e7f, entries=150, sequenceid=548, filesize=12.0 K 2024-11-09T06:16:53,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/49a3d83d18b64f95ae2d32bea0494fee as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/49a3d83d18b64f95ae2d32bea0494fee 2024-11-09T06:16:53,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/49a3d83d18b64f95ae2d32bea0494fee, entries=150, sequenceid=548, filesize=12.0 K 2024-11-09T06:16:53,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/a2789bd420fd49ad99a3c78a2eb9980f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/a2789bd420fd49ad99a3c78a2eb9980f 2024-11-09T06:16:53,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/a2789bd420fd49ad99a3c78a2eb9980f, entries=150, sequenceid=548, filesize=12.0 K 2024-11-09T06:16:53,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=13.42 KB/13740 for 96ddebd30416e9671cc930feb19be277 in 498ms, sequenceid=548, compaction requested=true 2024-11-09T06:16:53,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:53,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:16:53,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:53,883 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:53,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:16:53,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:53,883 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:53,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ddebd30416e9671cc930feb19be277:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:16:53,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:53,884 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:53,884 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:53,884 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/A is initiating minor compaction (all files) 2024-11-09T06:16:53,884 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/B is initiating minor compaction (all files) 2024-11-09T06:16:53,884 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/B in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:53,884 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/A in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:53,884 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f6b0395ee02e4044b05695ac6ffd5ff2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f951d6cfe3034ec986fa8d3e3c6f0de2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/126c8eac7eb54b36bce55eeb06101e7f] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=39.5 K 2024-11-09T06:16:53,884 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/b9446649f5bd4ca5b1d1b05304b911b0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/862f5c5eb92b4415ae35ecd433bdd1dd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/49a3d83d18b64f95ae2d32bea0494fee] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=37.1 K 2024-11-09T06:16:53,884 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b9446649f5bd4ca5b1d1b05304b911b0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731133009646 2024-11-09T06:16:53,884 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b0395ee02e4044b05695ac6ffd5ff2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731133009646 2024-11-09T06:16:53,885 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f951d6cfe3034ec986fa8d3e3c6f0de2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731133010823 2024-11-09T06:16:53,885 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 862f5c5eb92b4415ae35ecd433bdd1dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731133010823 2024-11-09T06:16:53,886 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 126c8eac7eb54b36bce55eeb06101e7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731133011291 2024-11-09T06:16:53,886 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a3d83d18b64f95ae2d32bea0494fee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731133011291 2024-11-09T06:16:53,899 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#B#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:53,899 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/82272b6f0d44441099400b9d52995ce3 is 50, key is test_row_0/B:col10/1731133011302/Put/seqid=0 2024-11-09T06:16:53,906 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#A#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:53,907 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/a5a266d622c84d988722db8e836c063b is 50, key is test_row_0/A:col10/1731133011302/Put/seqid=0 2024-11-09T06:16:53,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742171_1347 (size=13493) 2024-11-09T06:16:53,923 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/82272b6f0d44441099400b9d52995ce3 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/82272b6f0d44441099400b9d52995ce3 2024-11-09T06:16:53,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742172_1348 (size=13493) 2024-11-09T06:16:53,929 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/B of 96ddebd30416e9671cc930feb19be277 into 82272b6f0d44441099400b9d52995ce3(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:53,929 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:53,929 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/B, priority=13, startTime=1731133013883; duration=0sec 2024-11-09T06:16:53,929 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:16:53,929 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:B 2024-11-09T06:16:53,929 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:16:53,930 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:16:53,930 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 96ddebd30416e9671cc930feb19be277/C is initiating minor compaction (all files) 2024-11-09T06:16:53,930 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ddebd30416e9671cc930feb19be277/C in TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:53,930 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/317cca11f4474eb6bc158bcb980ba995, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2cada715f1a848db8c276f1e21af4ac7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/a2789bd420fd49ad99a3c78a2eb9980f] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp, totalSize=37.1 K 2024-11-09T06:16:53,930 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 317cca11f4474eb6bc158bcb980ba995, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731133009646 2024-11-09T06:16:53,930 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cada715f1a848db8c276f1e21af4ac7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731133010823 2024-11-09T06:16:53,931 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a2789bd420fd49ad99a3c78a2eb9980f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731133011291 2024-11-09T06:16:53,940 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ddebd30416e9671cc930feb19be277#C#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:16:53,940 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/53e70da2c62d4886bf7e549314b98e75 is 50, key is test_row_0/C:col10/1731133011302/Put/seqid=0 2024-11-09T06:16:53,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742173_1349 (size=13493) 2024-11-09T06:16:54,338 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/a5a266d622c84d988722db8e836c063b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/a5a266d622c84d988722db8e836c063b 2024-11-09T06:16:54,352 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/A of 96ddebd30416e9671cc930feb19be277 into a5a266d622c84d988722db8e836c063b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:54,353 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:54,353 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/A, priority=13, startTime=1731133013883; duration=0sec 2024-11-09T06:16:54,353 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:54,353 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:A 2024-11-09T06:16:54,368 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/53e70da2c62d4886bf7e549314b98e75 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/53e70da2c62d4886bf7e549314b98e75 2024-11-09T06:16:54,372 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ddebd30416e9671cc930feb19be277/C of 96ddebd30416e9671cc930feb19be277 into 53e70da2c62d4886bf7e549314b98e75(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:16:54,372 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ddebd30416e9671cc930feb19be277: 2024-11-09T06:16:54,372 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., storeName=96ddebd30416e9671cc930feb19be277/C, priority=13, startTime=1731133013883; duration=0sec 2024-11-09T06:16:54,372 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:16:54,372 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ddebd30416e9671cc930feb19be277:C 2024-11-09T06:16:56,884 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-09T06:16:56,884 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:56,884 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:56,884 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:56,884 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:56,894 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-09T06:16:56,895 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:16:56,895 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:56,895 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2658 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2682 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2756 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2680 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2669 2024-11-09T06:16:56,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T06:16:56,895 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T06:16:56,895 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T06:16:56,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:56,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:16:56,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:56,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:16:56,896 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T06:16:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T06:16:56,899 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133016899"}]},"ts":"1731133016899"} 2024-11-09T06:16:56,901 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T06:16:56,901 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T06:16:56,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T06:16:56,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ddebd30416e9671cc930feb19be277, UNASSIGN}] 2024-11-09T06:16:56,906 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ddebd30416e9671cc930feb19be277, UNASSIGN 2024-11-09T06:16:56,907 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=96ddebd30416e9671cc930feb19be277, regionState=CLOSING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:16:56,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ddebd30416e9671cc930feb19be277, UNASSIGN because future has completed 2024-11-09T06:16:56,908 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T06:16:56,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:16:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T06:16:57,062 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:57,062 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T06:16:57,062 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing 96ddebd30416e9671cc930feb19be277, disabling compactions & flushes 2024-11-09T06:16:57,062 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:57,062 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:57,062 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. after waiting 0 ms 2024-11-09T06:16:57,062 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:57,062 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing 96ddebd30416e9671cc930feb19be277 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-09T06:16:57,063 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=A 2024-11-09T06:16:57,063 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:57,063 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=B 2024-11-09T06:16:57,063 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:57,063 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96ddebd30416e9671cc930feb19be277, store=C 2024-11-09T06:16:57,063 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:16:57,067 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/623a371693c84af7b32af8441d46b4b9 is 50, key is test_row_0/A:col10/1731133013463/Put/seqid=0 2024-11-09T06:16:57,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742174_1350 (size=12301) 2024-11-09T06:16:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T06:16:57,299 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T06:16:57,472 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/623a371693c84af7b32af8441d46b4b9 2024-11-09T06:16:57,480 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/a150c6ff01054ce692c7ee170218d661 is 50, key is test_row_0/B:col10/1731133013463/Put/seqid=0 2024-11-09T06:16:57,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742175_1351 (size=12301) 2024-11-09T06:16:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T06:16:57,885 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/a150c6ff01054ce692c7ee170218d661 2024-11-09T06:16:57,891 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/d7d57a6eefd84edd875d86519add550c is 50, key is test_row_0/C:col10/1731133013463/Put/seqid=0 2024-11-09T06:16:57,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742176_1352 (size=12301) 2024-11-09T06:16:57,895 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/d7d57a6eefd84edd875d86519add550c 2024-11-09T06:16:57,899 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/A/623a371693c84af7b32af8441d46b4b9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/623a371693c84af7b32af8441d46b4b9 2024-11-09T06:16:57,903 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/623a371693c84af7b32af8441d46b4b9, entries=150, sequenceid=558, filesize=12.0 K 2024-11-09T06:16:57,904 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/B/a150c6ff01054ce692c7ee170218d661 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a150c6ff01054ce692c7ee170218d661 2024-11-09T06:16:57,907 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a150c6ff01054ce692c7ee170218d661, entries=150, sequenceid=558, filesize=12.0 K 2024-11-09T06:16:57,908 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/.tmp/C/d7d57a6eefd84edd875d86519add550c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/d7d57a6eefd84edd875d86519add550c 2024-11-09T06:16:57,911 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/d7d57a6eefd84edd875d86519add550c, entries=150, sequenceid=558, filesize=12.0 K 2024-11-09T06:16:57,912 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 96ddebd30416e9671cc930feb19be277 in 850ms, sequenceid=558, compaction requested=false 2024-11-09T06:16:57,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/2057bcd49a4140aaab4819ad2867a929, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/61d92dfc4ae14959b33a3ca1edead806, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/845067298f134c37875482090bd39d4f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/948c8254e1984faba9dc1d0f3c976150, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/96d66f018bcb438cbdc2fd8ae0d55f10, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/472324552ed5426193acd059f480c815, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/18dc9a1c05f8475b90d09abef2bd866b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f7ec945462ef4e74ad26dac4421b6006, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/b0c98206f0494b89b3174f8b64ac0329, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/485e0936e9d94c00a80877835f2088fa, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/74c87f5d71e548868a2f7cf58c8789c8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/bf85284361ce42778b050ae297691f4e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f0f61ee457fa47419490b7c30011d8d0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/30b11434204543649310ac1285a3bf10, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/c5218a989fbe4c5786e566846ee228c7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/72ad73e04f3945028e0bfa39987d8915, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6482bbca62db452a9be76249f279eaa1, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/0f2b9d70507c448aa4a3c5d2b351fab9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6314a5f1e88f4595af0277190b6a55ce, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/a8157a2aa2e04f3d85117c35e9c19fe4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/963928a9997c42e780207feec0c3bb69, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/02a5e40873de451ab8f50e6bf943465c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f9bd692ee2664a29b8884a41725e94b2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6b14eb091f1b421abe9684ddbe73ea61, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/ba5082247f3045b9908c8529fee7e050, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/4160a9cf223e46d2ad1fc2d4da07bf81, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/e53972cd2545450ab0faaec60390b54f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/be9bf9a4b1a54433880be5604efd6e8c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/e90388a2f66b48a982b1d4a5868b91a7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/66c248edff934bf38181c0e833233e7e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/d6d06ca4c71c441da8f842238cdb633e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/5fcd1a4466b044ca88d97db2afce509e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f6b0395ee02e4044b05695ac6ffd5ff2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f951d6cfe3034ec986fa8d3e3c6f0de2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/126c8eac7eb54b36bce55eeb06101e7f] to archive 2024-11-09T06:16:57,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:16:57,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/2057bcd49a4140aaab4819ad2867a929 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/2057bcd49a4140aaab4819ad2867a929 2024-11-09T06:16:57,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/61d92dfc4ae14959b33a3ca1edead806 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/61d92dfc4ae14959b33a3ca1edead806 2024-11-09T06:16:57,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/845067298f134c37875482090bd39d4f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/845067298f134c37875482090bd39d4f 2024-11-09T06:16:57,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/948c8254e1984faba9dc1d0f3c976150 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/948c8254e1984faba9dc1d0f3c976150 2024-11-09T06:16:57,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/96d66f018bcb438cbdc2fd8ae0d55f10 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/96d66f018bcb438cbdc2fd8ae0d55f10 2024-11-09T06:16:57,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/472324552ed5426193acd059f480c815 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/472324552ed5426193acd059f480c815 2024-11-09T06:16:57,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/18dc9a1c05f8475b90d09abef2bd866b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/18dc9a1c05f8475b90d09abef2bd866b 2024-11-09T06:16:57,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f7ec945462ef4e74ad26dac4421b6006 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f7ec945462ef4e74ad26dac4421b6006 2024-11-09T06:16:57,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/b0c98206f0494b89b3174f8b64ac0329 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/b0c98206f0494b89b3174f8b64ac0329 2024-11-09T06:16:57,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/485e0936e9d94c00a80877835f2088fa to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/485e0936e9d94c00a80877835f2088fa 2024-11-09T06:16:57,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/74c87f5d71e548868a2f7cf58c8789c8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/74c87f5d71e548868a2f7cf58c8789c8 2024-11-09T06:16:57,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/bf85284361ce42778b050ae297691f4e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/bf85284361ce42778b050ae297691f4e 2024-11-09T06:16:57,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f0f61ee457fa47419490b7c30011d8d0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f0f61ee457fa47419490b7c30011d8d0 2024-11-09T06:16:57,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/30b11434204543649310ac1285a3bf10 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/30b11434204543649310ac1285a3bf10 2024-11-09T06:16:57,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/c5218a989fbe4c5786e566846ee228c7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/c5218a989fbe4c5786e566846ee228c7 2024-11-09T06:16:57,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/72ad73e04f3945028e0bfa39987d8915 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/72ad73e04f3945028e0bfa39987d8915 2024-11-09T06:16:57,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6482bbca62db452a9be76249f279eaa1 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6482bbca62db452a9be76249f279eaa1 2024-11-09T06:16:57,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/0f2b9d70507c448aa4a3c5d2b351fab9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/0f2b9d70507c448aa4a3c5d2b351fab9 2024-11-09T06:16:57,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6314a5f1e88f4595af0277190b6a55ce to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6314a5f1e88f4595af0277190b6a55ce 2024-11-09T06:16:57,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/a8157a2aa2e04f3d85117c35e9c19fe4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/a8157a2aa2e04f3d85117c35e9c19fe4 2024-11-09T06:16:57,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/963928a9997c42e780207feec0c3bb69 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/963928a9997c42e780207feec0c3bb69 2024-11-09T06:16:57,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/02a5e40873de451ab8f50e6bf943465c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/02a5e40873de451ab8f50e6bf943465c 2024-11-09T06:16:57,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f9bd692ee2664a29b8884a41725e94b2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f9bd692ee2664a29b8884a41725e94b2 2024-11-09T06:16:57,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6b14eb091f1b421abe9684ddbe73ea61 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/6b14eb091f1b421abe9684ddbe73ea61 2024-11-09T06:16:57,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/ba5082247f3045b9908c8529fee7e050 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/ba5082247f3045b9908c8529fee7e050 2024-11-09T06:16:57,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/4160a9cf223e46d2ad1fc2d4da07bf81 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/4160a9cf223e46d2ad1fc2d4da07bf81 2024-11-09T06:16:57,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/e53972cd2545450ab0faaec60390b54f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/e53972cd2545450ab0faaec60390b54f 2024-11-09T06:16:57,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/be9bf9a4b1a54433880be5604efd6e8c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/be9bf9a4b1a54433880be5604efd6e8c 2024-11-09T06:16:57,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/e90388a2f66b48a982b1d4a5868b91a7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/e90388a2f66b48a982b1d4a5868b91a7 2024-11-09T06:16:57,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/66c248edff934bf38181c0e833233e7e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/66c248edff934bf38181c0e833233e7e 2024-11-09T06:16:57,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/d6d06ca4c71c441da8f842238cdb633e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/d6d06ca4c71c441da8f842238cdb633e 2024-11-09T06:16:57,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/5fcd1a4466b044ca88d97db2afce509e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/5fcd1a4466b044ca88d97db2afce509e 2024-11-09T06:16:57,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f6b0395ee02e4044b05695ac6ffd5ff2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f6b0395ee02e4044b05695ac6ffd5ff2 2024-11-09T06:16:57,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f951d6cfe3034ec986fa8d3e3c6f0de2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/f951d6cfe3034ec986fa8d3e3c6f0de2 2024-11-09T06:16:57,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/126c8eac7eb54b36bce55eeb06101e7f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/126c8eac7eb54b36bce55eeb06101e7f 2024-11-09T06:16:57,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/d1308d300f494e03b5fbc9f481267730, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/2a7bd850e682431c8ff2fc5cfc046e00, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/99d79cdabc18403489c61235873dc3d7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/dfcdf1aa2b804e3ba3cfd7298a73b01a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/95c5043b30ae490ea1dd92f43008c503, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4b29cffa2cc84974924fc7d7bd51a579, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/3d70a5a2e2b74c7ebdf6c8ca3394404d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/46dbcd475a1449f98dc0dfe2886c1efb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/95ab49ff525f4fdd8ec7b26cbd170f16, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4b1f5361b4044da0a96ada6fa8401ef9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/67ef569a0a1943dc84f22c4fbf08796e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4c8fc94a2ac14923bab920c44ebf89f1, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/01c2cc7ba1e541b4a3ad37c994282746, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/1a6c8fad6c174397b6db9e56df835e06, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/235c2af6b37b4f31a73c8f654dc6eaac, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/82665f34d08f4cd5aa6c6d83d28aa35f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/5ea981ef49174b95887870e6f195455b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/ed017d7e412a4ecbb04e49ba6f158f58, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/78a9f6d0a4184ea890d6b63dc314407c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a483cbade45c43aa9c17f768d54d7060, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/44332ed3ab974a9783361707d1710003, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/9af30a7616c148448568f537d97ea50c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/224e75aa7b5e493fb4c683b94e4c93bd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/30a971d556d54bb99ee1badaf218fbef, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/ffbc3a8dc39542599f3d251409686052, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/b175926f925c4eda91548a05853666ec, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/be3c8d74137040d096959c12a5a0c501, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/73b58b40e1a54da1bc17382104e096c4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/c4b7eb1d41ae40f999b7c0f2f5b0e233, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/6214fe533d0c4168a1f10d488d370a14, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a971f4414e8040b3976d967a89711ebf, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/b9446649f5bd4ca5b1d1b05304b911b0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/7fe6c1295aaa45fd9f1f52e4ac4526ff, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/862f5c5eb92b4415ae35ecd433bdd1dd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/49a3d83d18b64f95ae2d32bea0494fee] to archive 2024-11-09T06:16:57,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:16:57,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/d1308d300f494e03b5fbc9f481267730 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/d1308d300f494e03b5fbc9f481267730 2024-11-09T06:16:57,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/2a7bd850e682431c8ff2fc5cfc046e00 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/2a7bd850e682431c8ff2fc5cfc046e00 2024-11-09T06:16:57,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/99d79cdabc18403489c61235873dc3d7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/99d79cdabc18403489c61235873dc3d7 2024-11-09T06:16:57,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/dfcdf1aa2b804e3ba3cfd7298a73b01a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/dfcdf1aa2b804e3ba3cfd7298a73b01a 2024-11-09T06:16:57,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/95c5043b30ae490ea1dd92f43008c503 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/95c5043b30ae490ea1dd92f43008c503 2024-11-09T06:16:57,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4b29cffa2cc84974924fc7d7bd51a579 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4b29cffa2cc84974924fc7d7bd51a579 2024-11-09T06:16:57,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/3d70a5a2e2b74c7ebdf6c8ca3394404d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/3d70a5a2e2b74c7ebdf6c8ca3394404d 2024-11-09T06:16:57,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/46dbcd475a1449f98dc0dfe2886c1efb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/46dbcd475a1449f98dc0dfe2886c1efb 2024-11-09T06:16:57,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/95ab49ff525f4fdd8ec7b26cbd170f16 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/95ab49ff525f4fdd8ec7b26cbd170f16 2024-11-09T06:16:57,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4b1f5361b4044da0a96ada6fa8401ef9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4b1f5361b4044da0a96ada6fa8401ef9 2024-11-09T06:16:57,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/67ef569a0a1943dc84f22c4fbf08796e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/67ef569a0a1943dc84f22c4fbf08796e 2024-11-09T06:16:57,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4c8fc94a2ac14923bab920c44ebf89f1 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/4c8fc94a2ac14923bab920c44ebf89f1 2024-11-09T06:16:57,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/01c2cc7ba1e541b4a3ad37c994282746 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/01c2cc7ba1e541b4a3ad37c994282746 2024-11-09T06:16:57,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/1a6c8fad6c174397b6db9e56df835e06 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/1a6c8fad6c174397b6db9e56df835e06 2024-11-09T06:16:57,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/235c2af6b37b4f31a73c8f654dc6eaac to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/235c2af6b37b4f31a73c8f654dc6eaac 2024-11-09T06:16:57,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/82665f34d08f4cd5aa6c6d83d28aa35f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/82665f34d08f4cd5aa6c6d83d28aa35f 2024-11-09T06:16:57,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/5ea981ef49174b95887870e6f195455b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/5ea981ef49174b95887870e6f195455b 2024-11-09T06:16:57,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/ed017d7e412a4ecbb04e49ba6f158f58 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/ed017d7e412a4ecbb04e49ba6f158f58 2024-11-09T06:16:57,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/78a9f6d0a4184ea890d6b63dc314407c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/78a9f6d0a4184ea890d6b63dc314407c 2024-11-09T06:16:57,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a483cbade45c43aa9c17f768d54d7060 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a483cbade45c43aa9c17f768d54d7060 2024-11-09T06:16:57,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/44332ed3ab974a9783361707d1710003 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/44332ed3ab974a9783361707d1710003 2024-11-09T06:16:57,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/9af30a7616c148448568f537d97ea50c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/9af30a7616c148448568f537d97ea50c 2024-11-09T06:16:57,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/224e75aa7b5e493fb4c683b94e4c93bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/224e75aa7b5e493fb4c683b94e4c93bd 2024-11-09T06:16:57,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/30a971d556d54bb99ee1badaf218fbef to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/30a971d556d54bb99ee1badaf218fbef 2024-11-09T06:16:57,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/ffbc3a8dc39542599f3d251409686052 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/ffbc3a8dc39542599f3d251409686052 2024-11-09T06:16:57,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/b175926f925c4eda91548a05853666ec to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/b175926f925c4eda91548a05853666ec 2024-11-09T06:16:57,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/be3c8d74137040d096959c12a5a0c501 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/be3c8d74137040d096959c12a5a0c501 2024-11-09T06:16:57,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/73b58b40e1a54da1bc17382104e096c4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/73b58b40e1a54da1bc17382104e096c4 2024-11-09T06:16:57,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/c4b7eb1d41ae40f999b7c0f2f5b0e233 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/c4b7eb1d41ae40f999b7c0f2f5b0e233 2024-11-09T06:16:57,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/6214fe533d0c4168a1f10d488d370a14 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/6214fe533d0c4168a1f10d488d370a14 2024-11-09T06:16:57,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a971f4414e8040b3976d967a89711ebf to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a971f4414e8040b3976d967a89711ebf 2024-11-09T06:16:57,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/b9446649f5bd4ca5b1d1b05304b911b0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/b9446649f5bd4ca5b1d1b05304b911b0 2024-11-09T06:16:57,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/7fe6c1295aaa45fd9f1f52e4ac4526ff to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/7fe6c1295aaa45fd9f1f52e4ac4526ff 2024-11-09T06:16:57,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/862f5c5eb92b4415ae35ecd433bdd1dd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/862f5c5eb92b4415ae35ecd433bdd1dd 2024-11-09T06:16:57,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/49a3d83d18b64f95ae2d32bea0494fee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/49a3d83d18b64f95ae2d32bea0494fee 2024-11-09T06:16:58,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/93bd644863204668a46878c8db19b173, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/cee07c607cf44f38b76db494b0ae8f13, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/91449b4bf1514c58abc49d981eb830b0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/59a88d984b6a42aa99be056592e9f815, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0adebb1da0654136b287d2cd4885fa64, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/988e3130156e4897b2f346e14af59d1d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/961326ba0b2c4c5482c806203440a905, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/20b43f9659bf44e69113363dce0df27f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/47d87e07d88142c48a1a8257864cb34a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/7bc56cb3c5e34103ba08187c95fe16e8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/6c276864377342f383ec10801c1d1b52, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/8d4beafa00fd48f8b102109a955baa29, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/c276957f36d84d03a10b7525d760d15a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/109a85dd78234e9680e98b0ea7f77d85, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/75314851d46a4a0b8d71d9eb9c96dff5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/177dc52f722342038fa3f6a76b6e33a9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/5941b714fd694d74a7590caa067e7020, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/69f5f40527424de681ed8a2ed6f7abe0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/b483032f198e4be69861c0fcc0999688, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/bbd101819d26409abb314448bd2db5c6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0d5d0d61e06845cd860c51bca3460589, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/a9a042d555f44946bc3442df38aaf665, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/4c1df0142ba248f0997626d3367ed15c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/654e38b7998945188e9bc9ba86796938, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/8dd15ebe84e9470b957eb1b6c7d4539d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/e5f6e1ac1ba543b2a24301a4f0a3b866, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9c9461fb67fc4410953102a96992f4e7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/1b68a420dd4a47b2861d80ea80c83906, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/f5217709e01146c4855440e8844b7ea0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9466d2971bba4c4ea26eb39ba18fcbaf, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0de9730a24ea4f98a9ca3dd063002da4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/317cca11f4474eb6bc158bcb980ba995, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2e15227bfc534c9a80dce557bd7eb221, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2cada715f1a848db8c276f1e21af4ac7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/a2789bd420fd49ad99a3c78a2eb9980f] to archive 2024-11-09T06:16:58,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:16:58,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/93bd644863204668a46878c8db19b173 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/93bd644863204668a46878c8db19b173 2024-11-09T06:16:58,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/cee07c607cf44f38b76db494b0ae8f13 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/cee07c607cf44f38b76db494b0ae8f13 2024-11-09T06:16:58,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/91449b4bf1514c58abc49d981eb830b0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/91449b4bf1514c58abc49d981eb830b0 2024-11-09T06:16:58,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/59a88d984b6a42aa99be056592e9f815 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/59a88d984b6a42aa99be056592e9f815 2024-11-09T06:16:58,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0adebb1da0654136b287d2cd4885fa64 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0adebb1da0654136b287d2cd4885fa64 2024-11-09T06:16:58,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/988e3130156e4897b2f346e14af59d1d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/988e3130156e4897b2f346e14af59d1d 2024-11-09T06:16:58,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/961326ba0b2c4c5482c806203440a905 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/961326ba0b2c4c5482c806203440a905 2024-11-09T06:16:58,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/20b43f9659bf44e69113363dce0df27f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/20b43f9659bf44e69113363dce0df27f 2024-11-09T06:16:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T06:16:58,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/47d87e07d88142c48a1a8257864cb34a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/47d87e07d88142c48a1a8257864cb34a 2024-11-09T06:16:58,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/7bc56cb3c5e34103ba08187c95fe16e8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/7bc56cb3c5e34103ba08187c95fe16e8 2024-11-09T06:16:58,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/6c276864377342f383ec10801c1d1b52 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/6c276864377342f383ec10801c1d1b52 2024-11-09T06:16:58,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/8d4beafa00fd48f8b102109a955baa29 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/8d4beafa00fd48f8b102109a955baa29 2024-11-09T06:16:58,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/c276957f36d84d03a10b7525d760d15a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/c276957f36d84d03a10b7525d760d15a 2024-11-09T06:16:58,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/109a85dd78234e9680e98b0ea7f77d85 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/109a85dd78234e9680e98b0ea7f77d85 2024-11-09T06:16:58,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/75314851d46a4a0b8d71d9eb9c96dff5 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/75314851d46a4a0b8d71d9eb9c96dff5 2024-11-09T06:16:58,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/177dc52f722342038fa3f6a76b6e33a9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/177dc52f722342038fa3f6a76b6e33a9 2024-11-09T06:16:58,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/5941b714fd694d74a7590caa067e7020 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/5941b714fd694d74a7590caa067e7020 2024-11-09T06:16:58,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/69f5f40527424de681ed8a2ed6f7abe0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/69f5f40527424de681ed8a2ed6f7abe0 2024-11-09T06:16:58,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/b483032f198e4be69861c0fcc0999688 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/b483032f198e4be69861c0fcc0999688 2024-11-09T06:16:58,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/bbd101819d26409abb314448bd2db5c6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/bbd101819d26409abb314448bd2db5c6 2024-11-09T06:16:58,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0d5d0d61e06845cd860c51bca3460589 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0d5d0d61e06845cd860c51bca3460589 2024-11-09T06:16:58,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/a9a042d555f44946bc3442df38aaf665 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/a9a042d555f44946bc3442df38aaf665 2024-11-09T06:16:58,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/4c1df0142ba248f0997626d3367ed15c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/4c1df0142ba248f0997626d3367ed15c 2024-11-09T06:16:58,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/654e38b7998945188e9bc9ba86796938 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/654e38b7998945188e9bc9ba86796938 2024-11-09T06:16:58,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/8dd15ebe84e9470b957eb1b6c7d4539d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/8dd15ebe84e9470b957eb1b6c7d4539d 2024-11-09T06:16:58,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/e5f6e1ac1ba543b2a24301a4f0a3b866 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/e5f6e1ac1ba543b2a24301a4f0a3b866 2024-11-09T06:16:58,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9c9461fb67fc4410953102a96992f4e7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9c9461fb67fc4410953102a96992f4e7 2024-11-09T06:16:58,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/1b68a420dd4a47b2861d80ea80c83906 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/1b68a420dd4a47b2861d80ea80c83906 2024-11-09T06:16:58,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/f5217709e01146c4855440e8844b7ea0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/f5217709e01146c4855440e8844b7ea0 2024-11-09T06:16:58,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9466d2971bba4c4ea26eb39ba18fcbaf to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/9466d2971bba4c4ea26eb39ba18fcbaf 2024-11-09T06:16:58,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0de9730a24ea4f98a9ca3dd063002da4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/0de9730a24ea4f98a9ca3dd063002da4 2024-11-09T06:16:58,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/317cca11f4474eb6bc158bcb980ba995 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/317cca11f4474eb6bc158bcb980ba995 2024-11-09T06:16:58,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2e15227bfc534c9a80dce557bd7eb221 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2e15227bfc534c9a80dce557bd7eb221 2024-11-09T06:16:58,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2cada715f1a848db8c276f1e21af4ac7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/2cada715f1a848db8c276f1e21af4ac7 2024-11-09T06:16:58,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/a2789bd420fd49ad99a3c78a2eb9980f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/a2789bd420fd49ad99a3c78a2eb9980f 2024-11-09T06:16:58,095 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/recovered.edits/561.seqid, newMaxSeqId=561, maxSeqId=1 2024-11-09T06:16:58,096 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. 2024-11-09T06:16:58,096 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for 96ddebd30416e9671cc930feb19be277: Waiting for close lock at 1731133017062Running coprocessor pre-close hooks at 1731133017062Disabling compacts and flushes for region at 1731133017062Disabling writes for close at 1731133017062Obtaining lock to block concurrent updates at 1731133017062Preparing flush snapshotting stores in 96ddebd30416e9671cc930feb19be277 at 1731133017062Finished memstore snapshotting TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731133017063 (+1 ms)Flushing stores of TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277. at 1731133017063Flushing 96ddebd30416e9671cc930feb19be277/A: creating writer at 1731133017063Flushing 96ddebd30416e9671cc930feb19be277/A: appending metadata at 1731133017066 (+3 ms)Flushing 96ddebd30416e9671cc930feb19be277/A: closing flushed file at 1731133017066Flushing 96ddebd30416e9671cc930feb19be277/B: creating writer at 1731133017477 (+411 ms)Flushing 96ddebd30416e9671cc930feb19be277/B: appending metadata at 1731133017479 (+2 ms)Flushing 96ddebd30416e9671cc930feb19be277/B: closing flushed file at 1731133017479Flushing 96ddebd30416e9671cc930feb19be277/C: creating writer at 1731133017888 (+409 ms)Flushing 96ddebd30416e9671cc930feb19be277/C: appending metadata at 1731133017891 (+3 ms)Flushing 96ddebd30416e9671cc930feb19be277/C: closing flushed file at 1731133017891Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76877f4: reopening flushed file at 1731133017898 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@650705bb: reopening flushed file at 1731133017903 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54de824d: reopening flushed file at 1731133017907 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 96ddebd30416e9671cc930feb19be277 in 850ms, sequenceid=558, compaction requested=false at 1731133017912 (+5 ms)Writing region close event to WAL at 1731133018091 (+179 ms)Running coprocessor post-close hooks at 1731133018096 (+5 ms)Closed at 1731133018096 2024-11-09T06:16:58,098 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed 96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:58,099 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=96ddebd30416e9671cc930feb19be277, regionState=CLOSED 2024-11-09T06:16:58,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:16:58,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-09T06:16:58,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure 96ddebd30416e9671cc930feb19be277, server=4a84bcce7bad,46317,1731132929173 in 1.1940 sec 2024-11-09T06:16:58,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-09T06:16:58,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ddebd30416e9671cc930feb19be277, UNASSIGN in 1.1990 sec 2024-11-09T06:16:58,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-09T06:16:58,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2030 sec 2024-11-09T06:16:58,109 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133018109"}]},"ts":"1731133018109"} 2024-11-09T06:16:58,111 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T06:16:58,111 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T06:16:58,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2150 sec 2024-11-09T06:16:58,717 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T06:16:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T06:16:59,043 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:59,044 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T06:16:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:59,045 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:59,046 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T06:16:59,048 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:59,051 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/recovered.edits] 2024-11-09T06:16:59,054 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/623a371693c84af7b32af8441d46b4b9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/623a371693c84af7b32af8441d46b4b9 2024-11-09T06:16:59,055 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/a5a266d622c84d988722db8e836c063b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/A/a5a266d622c84d988722db8e836c063b 2024-11-09T06:16:59,057 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/82272b6f0d44441099400b9d52995ce3 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/82272b6f0d44441099400b9d52995ce3 2024-11-09T06:16:59,058 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a150c6ff01054ce692c7ee170218d661 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/B/a150c6ff01054ce692c7ee170218d661 2024-11-09T06:16:59,060 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/53e70da2c62d4886bf7e549314b98e75 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/53e70da2c62d4886bf7e549314b98e75 2024-11-09T06:16:59,061 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/d7d57a6eefd84edd875d86519add550c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/C/d7d57a6eefd84edd875d86519add550c 2024-11-09T06:16:59,063 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/recovered.edits/561.seqid to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277/recovered.edits/561.seqid 2024-11-09T06:16:59,064 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/96ddebd30416e9671cc930feb19be277 2024-11-09T06:16:59,064 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T06:16:59,066 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:59,070 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T06:16:59,072 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T06:16:59,073 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:59,073 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T06:16:59,074 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731133019073"}]},"ts":"9223372036854775807"} 2024-11-09T06:16:59,076 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T06:16:59,076 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 96ddebd30416e9671cc930feb19be277, NAME => 'TestAcidGuarantees,,1731132989987.96ddebd30416e9671cc930feb19be277.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T06:16:59,076 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T06:16:59,076 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731133019076"}]},"ts":"9223372036854775807"} 2024-11-09T06:16:59,078 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T06:16:59,078 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:59,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-09T06:16:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T06:16:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T06:16:59,155 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:16:59,163 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 252), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=805 (was 761) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1519 (was 731) - AvailableMemoryMB LEAK? - 2024-11-09T06:16:59,172 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=805, ProcessCount=11, AvailableMemoryMB=1519 2024-11-09T06:16:59,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T06:16:59,174 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T06:16:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T06:16:59,176 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T06:16:59,176 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:16:59,176 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-11-09T06:16:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T06:16:59,177 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T06:16:59,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742177_1353 (size=1061) 2024-11-09T06:16:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T06:16:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T06:16:59,585 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f 2024-11-09T06:16:59,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742178_1354 (size=53) 2024-11-09T06:16:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T06:16:59,994 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:16:59,994 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a2e6477244fddb2a7787090556d9c8bd, disabling compactions & flushes 2024-11-09T06:16:59,994 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:16:59,994 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:16:59,994 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. after waiting 0 ms 2024-11-09T06:16:59,994 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:16:59,994 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:16:59,994 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a2e6477244fddb2a7787090556d9c8bd: Waiting for close lock at 1731133019994Disabling compacts and flushes for region at 1731133019994Disabling writes for close at 1731133019994Writing region close event to WAL at 1731133019994Closed at 1731133019994 2024-11-09T06:16:59,996 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T06:16:59,996 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731133019996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731133019996"}]},"ts":"1731133019996"} 2024-11-09T06:16:59,997 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T06:16:59,998 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T06:16:59,999 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133019998"}]},"ts":"1731133019998"} 2024-11-09T06:17:00,000 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T06:17:00,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, ASSIGN}] 2024-11-09T06:17:00,002 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, ASSIGN 2024-11-09T06:17:00,002 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, ASSIGN; state=OFFLINE, location=4a84bcce7bad,46317,1731132929173; forceNewPlan=false, retain=false 2024-11-09T06:17:00,153 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a2e6477244fddb2a7787090556d9c8bd, regionState=OPENING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:00,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, ASSIGN because future has completed 2024-11-09T06:17:00,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:17:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T06:17:00,313 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:00,314 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} 2024-11-09T06:17:00,314 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,314 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:17:00,315 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,315 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,316 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,318 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:00,318 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2e6477244fddb2a7787090556d9c8bd columnFamilyName A 2024-11-09T06:17:00,318 DEBUG [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:00,319 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(327): Store=a2e6477244fddb2a7787090556d9c8bd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:00,319 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,320 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:00,320 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2e6477244fddb2a7787090556d9c8bd columnFamilyName B 2024-11-09T06:17:00,320 DEBUG [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:00,321 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(327): Store=a2e6477244fddb2a7787090556d9c8bd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:00,321 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,322 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:00,322 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2e6477244fddb2a7787090556d9c8bd columnFamilyName C 2024-11-09T06:17:00,322 DEBUG [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:00,322 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(327): Store=a2e6477244fddb2a7787090556d9c8bd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:00,322 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:00,322 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,323 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,323 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,324 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,324 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,325 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T06:17:00,326 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,327 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T06:17:00,328 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened a2e6477244fddb2a7787090556d9c8bd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65499730, jitterRate=-0.023977965116500854}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T06:17:00,328 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:00,329 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for a2e6477244fddb2a7787090556d9c8bd: Running coprocessor pre-open hook at 1731133020315Writing region info on filesystem at 1731133020315Initializing all the Stores at 1731133020316 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133020316Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133020316Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133020316Cleaning up temporary data from old regions at 1731133020324 (+8 ms)Running coprocessor post-open hooks at 1731133020328 (+4 ms)Region opened successfully at 1731133020329 (+1 ms) 2024-11-09T06:17:00,329 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., pid=83, masterSystemTime=1731133020309 2024-11-09T06:17:00,331 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:00,331 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:00,332 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a2e6477244fddb2a7787090556d9c8bd, regionState=OPEN, openSeqNum=2, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:00,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:17:00,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-09T06:17:00,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 in 178 msec 2024-11-09T06:17:00,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-09T06:17:00,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, ASSIGN in 336 msec 2024-11-09T06:17:00,338 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T06:17:00,338 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133020338"}]},"ts":"1731133020338"} 2024-11-09T06:17:00,340 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T06:17:00,341 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T06:17:00,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-09T06:17:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T06:17:01,323 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:01,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:01,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:01,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:01,325 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d8bb45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:01,328 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b21c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:01,329 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:01,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:01,332 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T06:17:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a84bcce7bad,37069,1731132928314 2024-11-09T06:17:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65aae795 2024-11-09T06:17:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T06:17:01,334 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T06:17:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T06:17:01,336 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T06:17:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:01,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742179_1355 (size=1097) 2024-11-09T06:17:01,755 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-09T06:17:01,755 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-09T06:17:01,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T06:17:01,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, REOPEN/MOVE}] 2024-11-09T06:17:01,760 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, REOPEN/MOVE 2024-11-09T06:17:01,761 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a2e6477244fddb2a7787090556d9c8bd, regionState=CLOSING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:01,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, REOPEN/MOVE because future has completed 2024-11-09T06:17:01,762 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T06:17:01,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:17:01,915 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:01,915 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T06:17:01,915 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing a2e6477244fddb2a7787090556d9c8bd, disabling compactions & flushes 2024-11-09T06:17:01,915 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:01,915 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:01,915 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. after waiting 0 ms 2024-11-09T06:17:01,915 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:01,919 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T06:17:01,920 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:01,920 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for a2e6477244fddb2a7787090556d9c8bd: Waiting for close lock at 1731133021915Running coprocessor pre-close hooks at 1731133021915Disabling compacts and flushes for region at 1731133021915Disabling writes for close at 1731133021915Writing region close event to WAL at 1731133021916 (+1 ms)Running coprocessor post-close hooks at 1731133021920 (+4 ms)Closed at 1731133021920 2024-11-09T06:17:01,920 WARN [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: a2e6477244fddb2a7787090556d9c8bd to self. 2024-11-09T06:17:01,922 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:01,923 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a2e6477244fddb2a7787090556d9c8bd, regionState=CLOSED 2024-11-09T06:17:01,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:17:01,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-09T06:17:01,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 in 163 msec 2024-11-09T06:17:01,928 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, REOPEN/MOVE; state=CLOSED, location=4a84bcce7bad,46317,1731132929173; forceNewPlan=false, retain=true 2024-11-09T06:17:02,079 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a2e6477244fddb2a7787090556d9c8bd, regionState=OPENING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, REOPEN/MOVE because future has completed 2024-11-09T06:17:02,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:17:02,236 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,236 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} 2024-11-09T06:17:02,237 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,237 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:17:02,237 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,237 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,239 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,240 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:02,240 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2e6477244fddb2a7787090556d9c8bd columnFamilyName A 2024-11-09T06:17:02,241 DEBUG [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:02,242 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(327): Store=a2e6477244fddb2a7787090556d9c8bd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:02,243 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,243 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:02,243 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2e6477244fddb2a7787090556d9c8bd columnFamilyName B 2024-11-09T06:17:02,243 DEBUG [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:02,244 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(327): Store=a2e6477244fddb2a7787090556d9c8bd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:02,244 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,244 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:02,245 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2e6477244fddb2a7787090556d9c8bd columnFamilyName C 2024-11-09T06:17:02,245 DEBUG [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:02,245 INFO [StoreOpener-a2e6477244fddb2a7787090556d9c8bd-1 {}] regionserver.HStore(327): Store=a2e6477244fddb2a7787090556d9c8bd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:02,245 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,245 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,246 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,247 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,248 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,248 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,248 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T06:17:02,249 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,250 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened a2e6477244fddb2a7787090556d9c8bd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72817009, jitterRate=0.08505798876285553}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T06:17:02,250 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,251 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for a2e6477244fddb2a7787090556d9c8bd: Running coprocessor pre-open hook at 1731133022237Writing region info on filesystem at 1731133022237Initializing all the Stores at 1731133022238 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133022238Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133022238Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133022238Cleaning up temporary data from old regions at 1731133022248 (+10 ms)Running coprocessor post-open hooks at 1731133022250 (+2 ms)Region opened successfully at 1731133022251 (+1 ms) 2024-11-09T06:17:02,251 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., pid=88, masterSystemTime=1731133022233 2024-11-09T06:17:02,253 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,253 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,254 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a2e6477244fddb2a7787090556d9c8bd, regionState=OPEN, openSeqNum=5, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:17:02,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-11-09T06:17:02,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 in 174 msec 2024-11-09T06:17:02,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-09T06:17:02,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, REOPEN/MOVE in 497 msec 2024-11-09T06:17:02,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-09T06:17:02,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 502 msec 2024-11-09T06:17:02,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 926 msec 2024-11-09T06:17:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T06:17:02,264 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:02,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:02,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:02,266 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6081f862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:02,268 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d52166e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:02,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:02,270 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:02,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:02,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395e04b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:02,272 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:02,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a2b304f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:02,273 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:02,274 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a48c031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:02,276 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c008c52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:02,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:02,278 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135407e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:02,280 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7058296e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:02,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:02,282 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a07b55e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:02,283 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57cd934b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:02,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:02,285 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5219bc92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:02,287 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:02,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c2f65fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,288 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:02,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:02,289 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:02,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:02,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:02,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65bd4ddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:02,290 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c51d5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:02,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:02,293 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43c44e1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:02,294 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f6ab570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:02,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:02,297 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b5bb9f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:02,298 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:02,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b5935e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:02,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:02,301 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d88e69f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:02,303 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d74c69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:02,308 DEBUG [Thread-1618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:02,308 DEBUG [Thread-1624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:02,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:02,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:02,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:02,321 DEBUG [Thread-1626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:02,325 DEBUG [Thread-1622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:02,326 DEBUG [Thread-1609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:02,326 DEBUG [Thread-1620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:02,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:02,326 DEBUG [Thread-1607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:02,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:02,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:02,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:02,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:02,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:02,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:02,333 DEBUG [Thread-1611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:02,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:02,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:02,335 DEBUG [Thread-1613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:02,336 DEBUG [Thread-1615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:02,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:02,337 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:02,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:02,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:02,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:02,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-09T06:17:02,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:02,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:02,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T06:17:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:02,348 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:02,350 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:02,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T06:17:02,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:02,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:02,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:02,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:02,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:02,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:02,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097f446c0f2c2f43d9beea81cb068a4bae_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133022355/Put/seqid=0 2024-11-09T06:17:02,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731133022390, max timestamp=1731133022437Num uniques 859; 2024-11-09T06:17:02,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731133022390, max timestamp=1731133022437Num uniques 859; 2024-11-09T06:17:02,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T06:17:02,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T06:17:02,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133022425, max timestamp=1731133022437Num uniques 859; 2024-11-09T06:17:02,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:17:02,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742180_1356 (size=17034) 2024-11-09T06:17:02,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T06:17:02,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133082486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133082488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T06:17:02,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133082489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133082491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133082491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133082602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133082602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133082613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133082614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133082622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T06:17:02,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:02,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T06:17:02,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T06:17:02,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:02,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133082834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133082835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133082835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133082835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,847 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:02,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133082842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:02,857 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097f446c0f2c2f43d9beea81cb068a4bae_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097f446c0f2c2f43d9beea81cb068a4bae_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:02,859 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/9d50dca1177348a8967b954f296f42dd, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:02,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/9d50dca1177348a8967b954f296f42dd is 175, key is test_row_0/A:col10/1731133022355/Put/seqid=0 2024-11-09T06:17:02,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742181_1357 (size=48139) 2024-11-09T06:17:02,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T06:17:02,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:02,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:02,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T06:17:03,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T06:17:03,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:03,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:03,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:03,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:03,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:03,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133083152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133083152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133083153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133083153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133083153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/9d50dca1177348a8967b954f296f42dd 2024-11-09T06:17:03,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T06:17:03,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:03,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:03,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:03,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:03,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:03,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/b4589f3b009345bfa00497706a516c75 is 50, key is test_row_0/B:col10/1731133022355/Put/seqid=0 2024-11-09T06:17:03,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742182_1358 (size=12001) 2024-11-09T06:17:03,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/b4589f3b009345bfa00497706a516c75 2024-11-09T06:17:03,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/02bccfae5115403f86a6f83eb8e1df84 is 50, key is test_row_0/C:col10/1731133022355/Put/seqid=0 2024-11-09T06:17:03,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742183_1359 (size=12001) 2024-11-09T06:17:03,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/02bccfae5115403f86a6f83eb8e1df84 2024-11-09T06:17:03,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/9d50dca1177348a8967b954f296f42dd as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9d50dca1177348a8967b954f296f42dd 2024-11-09T06:17:03,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9d50dca1177348a8967b954f296f42dd, entries=250, sequenceid=15, filesize=47.0 K 2024-11-09T06:17:03,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/b4589f3b009345bfa00497706a516c75 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b4589f3b009345bfa00497706a516c75 2024-11-09T06:17:03,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T06:17:03,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:03,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:03,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:03,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:03,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:03,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b4589f3b009345bfa00497706a516c75, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T06:17:03,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/02bccfae5115403f86a6f83eb8e1df84 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/02bccfae5115403f86a6f83eb8e1df84 2024-11-09T06:17:03,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/02bccfae5115403f86a6f83eb8e1df84, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T06:17:03,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for a2e6477244fddb2a7787090556d9c8bd in 1084ms, sequenceid=15, compaction requested=false 2024-11-09T06:17:03,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T06:17:03,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T06:17:03,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:03,587 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-09T06:17:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:03,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:03,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:03,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:03,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110926b985f4b4124d9f8d73f62baeaa724c_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133022487/Put/seqid=0 2024-11-09T06:17:03,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742184_1360 (size=12154) 2024-11-09T06:17:03,664 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T06:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:03,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:03,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133083685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133083697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133083699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133083703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133083704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133083813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133083813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133083813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133083822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:03,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133083833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:03,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:03,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:04,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133084023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:04,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133084034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:04,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133084034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:04,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133084043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:04,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133084052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:04,068 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110926b985f4b4124d9f8d73f62baeaa724c_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110926b985f4b4124d9f8d73f62baeaa724c_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:04,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/02af962123cf49178ab64275c3013f49, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:04,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/02af962123cf49178ab64275c3013f49 is 175, key is test_row_0/A:col10/1731133022487/Put/seqid=0 2024-11-09T06:17:04,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742185_1361 (size=30955) 2024-11-09T06:17:04,098 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/02af962123cf49178ab64275c3013f49 2024-11-09T06:17:04,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/9415b960df3e42d6b6da871f03f0d2d2 is 50, key is test_row_0/B:col10/1731133022487/Put/seqid=0 2024-11-09T06:17:04,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742186_1362 (size=12001) 2024-11-09T06:17:04,117 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/9415b960df3e42d6b6da871f03f0d2d2 2024-11-09T06:17:04,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/66c2616ad0ee4904a88915d220d568a5 is 50, key is test_row_0/C:col10/1731133022487/Put/seqid=0 2024-11-09T06:17:04,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742187_1363 (size=12001) 2024-11-09T06:17:04,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133084342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133084343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:04,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133084343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:04,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133084363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:04,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133084373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T06:17:04,540 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/66c2616ad0ee4904a88915d220d568a5 2024-11-09T06:17:04,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/02af962123cf49178ab64275c3013f49 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/02af962123cf49178ab64275c3013f49 2024-11-09T06:17:04,567 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/02af962123cf49178ab64275c3013f49, entries=150, sequenceid=50, filesize=30.2 K 2024-11-09T06:17:04,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/9415b960df3e42d6b6da871f03f0d2d2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/9415b960df3e42d6b6da871f03f0d2d2 2024-11-09T06:17:04,574 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/9415b960df3e42d6b6da871f03f0d2d2, entries=150, sequenceid=50, filesize=11.7 K 2024-11-09T06:17:04,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/66c2616ad0ee4904a88915d220d568a5 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/66c2616ad0ee4904a88915d220d568a5 2024-11-09T06:17:04,588 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/66c2616ad0ee4904a88915d220d568a5, entries=150, sequenceid=50, filesize=11.7 K 2024-11-09T06:17:04,589 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for a2e6477244fddb2a7787090556d9c8bd in 1003ms, sequenceid=50, compaction requested=false 2024-11-09T06:17:04,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:04,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:04,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-09T06:17:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-09T06:17:04,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-09T06:17:04,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2410 sec 2024-11-09T06:17:04,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.2580 sec 2024-11-09T06:17:04,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:04,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T06:17:04,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:04,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:04,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:04,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:04,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:04,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:04,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109007f1e4d4f164014b550493ec52d8cda_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133024863/Put/seqid=0 2024-11-09T06:17:04,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742188_1364 (size=14594) 2024-11-09T06:17:04,907 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:04,913 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109007f1e4d4f164014b550493ec52d8cda_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109007f1e4d4f164014b550493ec52d8cda_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:04,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7ea9902e227244fc9f538edecdd7c0a1, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:04,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7ea9902e227244fc9f538edecdd7c0a1 is 175, key is test_row_0/A:col10/1731133024863/Put/seqid=0 2024-11-09T06:17:04,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133024904, max timestamp=1731133024922Num uniques 859; 2024-11-09T06:17:04,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:04,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133024904, max timestamp=1731133024925Num uniques 859; 2024-11-09T06:17:04,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:04,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133024904, max timestamp=1731133024925Num uniques 859; 2024-11-09T06:17:04,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:04,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742189_1365 (size=39549) 2024-11-09T06:17:04,938 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7ea9902e227244fc9f538edecdd7c0a1 2024-11-09T06:17:04,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/8e0723249d1e4f8e9c87d2d426d0e0a0 is 50, key is test_row_0/B:col10/1731133024863/Put/seqid=0 2024-11-09T06:17:04,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742190_1366 (size=12001) 2024-11-09T06:17:04,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/8e0723249d1e4f8e9c87d2d426d0e0a0 2024-11-09T06:17:04,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/85ca7d9aa5994d4387fadd12f30f8015 is 50, key is test_row_0/C:col10/1731133024863/Put/seqid=0 2024-11-09T06:17:04,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133084982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133084988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742191_1367 (size=12001) 2024-11-09T06:17:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133084990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133084991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133084990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/85ca7d9aa5994d4387fadd12f30f8015 2024-11-09T06:17:05,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7ea9902e227244fc9f538edecdd7c0a1 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7ea9902e227244fc9f538edecdd7c0a1 2024-11-09T06:17:05,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7ea9902e227244fc9f538edecdd7c0a1, entries=200, sequenceid=61, filesize=38.6 K 2024-11-09T06:17:05,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/8e0723249d1e4f8e9c87d2d426d0e0a0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8e0723249d1e4f8e9c87d2d426d0e0a0 2024-11-09T06:17:05,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8e0723249d1e4f8e9c87d2d426d0e0a0, entries=150, sequenceid=61, filesize=11.7 K 2024-11-09T06:17:05,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/85ca7d9aa5994d4387fadd12f30f8015 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/85ca7d9aa5994d4387fadd12f30f8015 2024-11-09T06:17:05,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/85ca7d9aa5994d4387fadd12f30f8015, entries=150, sequenceid=61, filesize=11.7 K 2024-11-09T06:17:05,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a2e6477244fddb2a7787090556d9c8bd in 174ms, sequenceid=61, compaction requested=true 2024-11-09T06:17:05,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:05,038 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:05,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:05,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:05,039 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:05,040 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:05,040 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/A is initiating minor compaction (all files) 2024-11-09T06:17:05,040 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/A in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:05,040 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9d50dca1177348a8967b954f296f42dd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/02af962123cf49178ab64275c3013f49, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7ea9902e227244fc9f538edecdd7c0a1] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=115.9 K 2024-11-09T06:17:05,040 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:05,040 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9d50dca1177348a8967b954f296f42dd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/02af962123cf49178ab64275c3013f49, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7ea9902e227244fc9f538edecdd7c0a1] 2024-11-09T06:17:05,040 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:05,040 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/B is initiating minor compaction (all files) 2024-11-09T06:17:05,040 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/B in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:05,041 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b4589f3b009345bfa00497706a516c75, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/9415b960df3e42d6b6da871f03f0d2d2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8e0723249d1e4f8e9c87d2d426d0e0a0] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=35.2 K 2024-11-09T06:17:05,041 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d50dca1177348a8967b954f296f42dd, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731133022343 2024-11-09T06:17:05,041 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b4589f3b009345bfa00497706a516c75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731133022343 2024-11-09T06:17:05,041 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02af962123cf49178ab64275c3013f49, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731133022486 2024-11-09T06:17:05,041 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9415b960df3e42d6b6da871f03f0d2d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731133022486 2024-11-09T06:17:05,041 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ea9902e227244fc9f538edecdd7c0a1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731133023679 2024-11-09T06:17:05,041 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e0723249d1e4f8e9c87d2d426d0e0a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731133023696 2024-11-09T06:17:05,070 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#B#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:05,070 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/7338403a4d8f4fb5a90e0defd45f0f35 is 50, key is test_row_0/B:col10/1731133024863/Put/seqid=0 2024-11-09T06:17:05,077 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:05,097 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109e2ebe3143c1d46ac9492e594a050dcfc_a2e6477244fddb2a7787090556d9c8bd store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:05,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742192_1368 (size=12104) 2024-11-09T06:17:05,100 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109e2ebe3143c1d46ac9492e594a050dcfc_a2e6477244fddb2a7787090556d9c8bd, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:05,100 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e2ebe3143c1d46ac9492e594a050dcfc_a2e6477244fddb2a7787090556d9c8bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:05,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-09T06:17:05,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:05,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:05,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:05,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:05,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:05,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:05,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742193_1369 (size=4469) 2024-11-09T06:17:05,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133085131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133085138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133085140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e6470981a53b489fb86b4e9c64d77208_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133024985/Put/seqid=0 2024-11-09T06:17:05,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133085147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133085148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742194_1370 (size=14594) 2024-11-09T06:17:05,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133085253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133085253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133085262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133085263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133085264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133085472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133085472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133085483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133085483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,502 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/7338403a4d8f4fb5a90e0defd45f0f35 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/7338403a4d8f4fb5a90e0defd45f0f35 2024-11-09T06:17:05,506 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/B of a2e6477244fddb2a7787090556d9c8bd into 7338403a4d8f4fb5a90e0defd45f0f35(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:05,506 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:05,506 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/B, priority=13, startTime=1731133025038; duration=0sec 2024-11-09T06:17:05,506 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:05,506 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:B 2024-11-09T06:17:05,506 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:05,507 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:05,507 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/C is initiating minor compaction (all files) 2024-11-09T06:17:05,507 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/C in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:05,508 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/02bccfae5115403f86a6f83eb8e1df84, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/66c2616ad0ee4904a88915d220d568a5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/85ca7d9aa5994d4387fadd12f30f8015] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=35.2 K 2024-11-09T06:17:05,508 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 02bccfae5115403f86a6f83eb8e1df84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731133022343 2024-11-09T06:17:05,508 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 66c2616ad0ee4904a88915d220d568a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731133022486 2024-11-09T06:17:05,508 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 85ca7d9aa5994d4387fadd12f30f8015, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731133023696 2024-11-09T06:17:05,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133085492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,522 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#C#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:05,522 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/9d2058c8f8364c34b7b4b255cc034a89 is 50, key is test_row_0/C:col10/1731133024863/Put/seqid=0 2024-11-09T06:17:05,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742195_1371 (size=12104) 2024-11-09T06:17:05,546 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#A#compaction#315 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:05,547 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/9966ad1556d3482b846e86584ead59ec is 175, key is test_row_0/A:col10/1731133024863/Put/seqid=0 2024-11-09T06:17:05,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742196_1372 (size=31058) 2024-11-09T06:17:05,592 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:05,595 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e6470981a53b489fb86b4e9c64d77208_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e6470981a53b489fb86b4e9c64d77208_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:05,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/a80f317e3b36488d9fe262edf81ef3fe, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:05,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/a80f317e3b36488d9fe262edf81ef3fe is 175, key is test_row_0/A:col10/1731133024985/Put/seqid=0 2024-11-09T06:17:05,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742197_1373 (size=39549) 2024-11-09T06:17:05,600 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/a80f317e3b36488d9fe262edf81ef3fe 2024-11-09T06:17:05,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/b35dfa42198d424780e5934eb38cd193 is 50, key is test_row_0/B:col10/1731133024985/Put/seqid=0 2024-11-09T06:17:05,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742198_1374 (size=12001) 2024-11-09T06:17:05,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/b35dfa42198d424780e5934eb38cd193 2024-11-09T06:17:05,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/2eb0b4931a9745cc97dc9c8793276b35 is 50, key is test_row_0/C:col10/1731133024985/Put/seqid=0 2024-11-09T06:17:05,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742199_1375 (size=12001) 2024-11-09T06:17:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133085792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133085803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133085803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133085803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133085812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:05,934 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/9d2058c8f8364c34b7b4b255cc034a89 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/9d2058c8f8364c34b7b4b255cc034a89 2024-11-09T06:17:05,938 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/C of a2e6477244fddb2a7787090556d9c8bd into 9d2058c8f8364c34b7b4b255cc034a89(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:05,938 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:05,938 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/C, priority=13, startTime=1731133025039; duration=0sec 2024-11-09T06:17:05,938 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:05,939 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:C 2024-11-09T06:17:05,957 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/9966ad1556d3482b846e86584ead59ec as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9966ad1556d3482b846e86584ead59ec 2024-11-09T06:17:05,962 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/A of a2e6477244fddb2a7787090556d9c8bd into 9966ad1556d3482b846e86584ead59ec(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:05,962 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:05,962 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/A, priority=13, startTime=1731133025038; duration=0sec 2024-11-09T06:17:05,962 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:05,962 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:A 2024-11-09T06:17:06,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/2eb0b4931a9745cc97dc9c8793276b35 2024-11-09T06:17:06,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/a80f317e3b36488d9fe262edf81ef3fe as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/a80f317e3b36488d9fe262edf81ef3fe 2024-11-09T06:17:06,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/a80f317e3b36488d9fe262edf81ef3fe, entries=200, sequenceid=97, filesize=38.6 K 2024-11-09T06:17:06,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/b35dfa42198d424780e5934eb38cd193 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b35dfa42198d424780e5934eb38cd193 2024-11-09T06:17:06,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b35dfa42198d424780e5934eb38cd193, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T06:17:06,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/2eb0b4931a9745cc97dc9c8793276b35 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/2eb0b4931a9745cc97dc9c8793276b35 2024-11-09T06:17:06,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/2eb0b4931a9745cc97dc9c8793276b35, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T06:17:06,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a2e6477244fddb2a7787090556d9c8bd in 944ms, sequenceid=97, compaction requested=false 2024-11-09T06:17:06,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:06,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T06:17:06,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:06,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:06,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:06,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:06,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:06,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:06,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109464d41c2b84b4d7e8aef5bdbdc80ba1f_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133025136/Put/seqid=0 2024-11-09T06:17:06,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742200_1376 (size=14594) 2024-11-09T06:17:06,336 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:06,342 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109464d41c2b84b4d7e8aef5bdbdc80ba1f_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109464d41c2b84b4d7e8aef5bdbdc80ba1f_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:06,343 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7e68d661ea8546a7b611dd8ab1c8a43f, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:06,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7e68d661ea8546a7b611dd8ab1c8a43f is 175, key is test_row_0/A:col10/1731133025136/Put/seqid=0 2024-11-09T06:17:06,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742201_1377 (size=39549) 2024-11-09T06:17:06,350 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7e68d661ea8546a7b611dd8ab1c8a43f 2024-11-09T06:17:06,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/0539e6adae4641348b4b71cefc17944f is 50, key is test_row_0/B:col10/1731133025136/Put/seqid=0 2024-11-09T06:17:06,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742202_1378 (size=12001) 2024-11-09T06:17:06,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133086360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731133026346, max timestamp=1731133026373Num uniques 859; 2024-11-09T06:17:06,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731133026346, max timestamp=1731133026373Num uniques 859; 2024-11-09T06:17:06,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:17:06,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:17:06,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731133026346, max timestamp=1731133026373Num uniques 859; 2024-11-09T06:17:06,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T06:17:06,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133086427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133086430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133086431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133086446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133086483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T06:17:06,503 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:06,504 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-09T06:17:06,506 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T06:17:06,507 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:06,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133086555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133086563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133086563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133086563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T06:17:06,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-09T06:17:06,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:06,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:06,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:06,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:06,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:06,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133086703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/0539e6adae4641348b4b71cefc17944f 2024-11-09T06:17:06,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/e4cfb9cb892d4380b569206a88a1ca52 is 50, key is test_row_0/C:col10/1731133025136/Put/seqid=0 2024-11-09T06:17:06,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742203_1379 (size=12001) 2024-11-09T06:17:06,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/e4cfb9cb892d4380b569206a88a1ca52 2024-11-09T06:17:06,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133086782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133086782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133086783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133086783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:06,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7e68d661ea8546a7b611dd8ab1c8a43f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7e68d661ea8546a7b611dd8ab1c8a43f 2024-11-09T06:17:06,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7e68d661ea8546a7b611dd8ab1c8a43f, entries=200, sequenceid=111, filesize=38.6 K 2024-11-09T06:17:06,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/0539e6adae4641348b4b71cefc17944f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/0539e6adae4641348b4b71cefc17944f 2024-11-09T06:17:06,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/0539e6adae4641348b4b71cefc17944f, entries=150, sequenceid=111, filesize=11.7 K 2024-11-09T06:17:06,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/e4cfb9cb892d4380b569206a88a1ca52 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/e4cfb9cb892d4380b569206a88a1ca52 2024-11-09T06:17:06,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-09T06:17:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:06,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:06,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/e4cfb9cb892d4380b569206a88a1ca52, entries=150, sequenceid=111, filesize=11.7 K 2024-11-09T06:17:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T06:17:06,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a2e6477244fddb2a7787090556d9c8bd in 516ms, sequenceid=111, compaction requested=true 2024-11-09T06:17:06,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:06,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:06,823 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:06,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:06,824 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:06,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:06,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:06,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:06,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:06,825 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:06,825 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/B is initiating minor compaction (all files) 2024-11-09T06:17:06,825 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/B in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:06,825 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/7338403a4d8f4fb5a90e0defd45f0f35, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b35dfa42198d424780e5934eb38cd193, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/0539e6adae4641348b4b71cefc17944f] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=35.3 K 2024-11-09T06:17:06,826 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:06,826 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/A is initiating minor compaction (all files) 2024-11-09T06:17:06,826 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7338403a4d8f4fb5a90e0defd45f0f35, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731133023696 2024-11-09T06:17:06,826 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/A in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:06,826 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9966ad1556d3482b846e86584ead59ec, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/a80f317e3b36488d9fe262edf81ef3fe, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7e68d661ea8546a7b611dd8ab1c8a43f] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=107.6 K 2024-11-09T06:17:06,827 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:06,827 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9966ad1556d3482b846e86584ead59ec, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/a80f317e3b36488d9fe262edf81ef3fe, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7e68d661ea8546a7b611dd8ab1c8a43f] 2024-11-09T06:17:06,827 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9966ad1556d3482b846e86584ead59ec, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731133023696 2024-11-09T06:17:06,827 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b35dfa42198d424780e5934eb38cd193, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731133024980 2024-11-09T06:17:06,828 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a80f317e3b36488d9fe262edf81ef3fe, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731133024980 2024-11-09T06:17:06,831 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0539e6adae4641348b4b71cefc17944f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731133025136 2024-11-09T06:17:06,831 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e68d661ea8546a7b611dd8ab1c8a43f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731133025136 2024-11-09T06:17:06,837 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:06,838 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#B#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:06,838 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/20abcb8b3e12466dadb218106a7557b8 is 50, key is test_row_0/B:col10/1731133025136/Put/seqid=0 2024-11-09T06:17:06,843 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109ada8161e28844ffda24f655e1cbe097f_a2e6477244fddb2a7787090556d9c8bd store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:06,845 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109ada8161e28844ffda24f655e1cbe097f_a2e6477244fddb2a7787090556d9c8bd, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:06,845 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ada8161e28844ffda24f655e1cbe097f_a2e6477244fddb2a7787090556d9c8bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:06,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742204_1380 (size=12207) 2024-11-09T06:17:06,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742205_1381 (size=4469) 2024-11-09T06:17:06,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-09T06:17:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:06,969 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-09T06:17:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e0c04dfa2b3d4a068a47d11cc0cecb86_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133026427/Put/seqid=0 2024-11-09T06:17:06,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742206_1382 (size=12304) 2024-11-09T06:17:07,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:07,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133087104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133087104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133087104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133087105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133087107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T06:17:07,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133087233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,257 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/20abcb8b3e12466dadb218106a7557b8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/20abcb8b3e12466dadb218106a7557b8 2024-11-09T06:17:07,262 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/B of a2e6477244fddb2a7787090556d9c8bd into 20abcb8b3e12466dadb218106a7557b8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:07,262 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:07,262 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/B, priority=13, startTime=1731133026823; duration=0sec 2024-11-09T06:17:07,262 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:07,262 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:B 2024-11-09T06:17:07,262 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:07,263 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:07,263 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/C is initiating minor compaction (all files) 2024-11-09T06:17:07,263 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/C in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:07,263 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/9d2058c8f8364c34b7b4b255cc034a89, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/2eb0b4931a9745cc97dc9c8793276b35, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/e4cfb9cb892d4380b569206a88a1ca52] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=35.3 K 2024-11-09T06:17:07,263 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d2058c8f8364c34b7b4b255cc034a89, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731133023696 2024-11-09T06:17:07,264 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eb0b4931a9745cc97dc9c8793276b35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731133024980 2024-11-09T06:17:07,264 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting e4cfb9cb892d4380b569206a88a1ca52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731133025136 2024-11-09T06:17:07,271 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#C#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:07,271 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/86dce45162f34844ac54dee1258b0525 is 50, key is test_row_0/C:col10/1731133025136/Put/seqid=0 2024-11-09T06:17:07,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742207_1383 (size=12207) 2024-11-09T06:17:07,275 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#A#compaction#324 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:07,276 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/51871e6623ac43569435bddd62e00e40 is 175, key is test_row_0/A:col10/1731133025136/Put/seqid=0 2024-11-09T06:17:07,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742208_1384 (size=31161) 2024-11-09T06:17:07,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:07,401 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e0c04dfa2b3d4a068a47d11cc0cecb86_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e0c04dfa2b3d4a068a47d11cc0cecb86_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:07,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7f4876a08df14e288718948ce8f3ec0b, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:07,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7f4876a08df14e288718948ce8f3ec0b is 175, key is test_row_0/A:col10/1731133026427/Put/seqid=0 2024-11-09T06:17:07,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742209_1385 (size=31105) 2024-11-09T06:17:07,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133087453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133087623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133087623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133087623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133087623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T06:17:07,679 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/86dce45162f34844ac54dee1258b0525 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/86dce45162f34844ac54dee1258b0525 2024-11-09T06:17:07,686 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/C of a2e6477244fddb2a7787090556d9c8bd into 86dce45162f34844ac54dee1258b0525(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:07,686 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:07,686 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/C, priority=13, startTime=1731133026824; duration=0sec 2024-11-09T06:17:07,686 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:07,686 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:C 2024-11-09T06:17:07,696 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/51871e6623ac43569435bddd62e00e40 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/51871e6623ac43569435bddd62e00e40 2024-11-09T06:17:07,706 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/A of a2e6477244fddb2a7787090556d9c8bd into 51871e6623ac43569435bddd62e00e40(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:07,706 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:07,706 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/A, priority=13, startTime=1731133026823; duration=0sec 2024-11-09T06:17:07,707 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:07,707 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:A 2024-11-09T06:17:07,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133087773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:07,824 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7f4876a08df14e288718948ce8f3ec0b 2024-11-09T06:17:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/08027f7e656548e1a20404a15a5e8a17 is 50, key is test_row_0/B:col10/1731133026427/Put/seqid=0 2024-11-09T06:17:07,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742210_1386 (size=12151) 2024-11-09T06:17:08,238 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/08027f7e656548e1a20404a15a5e8a17 2024-11-09T06:17:08,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/ef01a8ddd40b43548dec1b38fec797d0 is 50, key is test_row_0/C:col10/1731133026427/Put/seqid=0 2024-11-09T06:17:08,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742211_1387 (size=12151) 2024-11-09T06:17:08,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133088292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:08,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133088642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:08,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133088643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:08,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133088643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:08,654 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:08,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:08,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:08,654 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:08,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:08,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:08,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:08,654 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_0' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:08,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:08,658 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/ef01a8ddd40b43548dec1b38fec797d0 2024-11-09T06:17:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133088653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:08,659 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_0' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T06:17:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7f4876a08df14e288718948ce8f3ec0b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7f4876a08df14e288718948ce8f3ec0b 2024-11-09T06:17:08,677 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7f4876a08df14e288718948ce8f3ec0b, entries=150, sequenceid=145, filesize=30.4 K 2024-11-09T06:17:08,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/08027f7e656548e1a20404a15a5e8a17 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/08027f7e656548e1a20404a15a5e8a17 2024-11-09T06:17:08,692 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/08027f7e656548e1a20404a15a5e8a17, entries=150, sequenceid=145, filesize=11.9 K 2024-11-09T06:17:08,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/ef01a8ddd40b43548dec1b38fec797d0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/ef01a8ddd40b43548dec1b38fec797d0 2024-11-09T06:17:08,701 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/ef01a8ddd40b43548dec1b38fec797d0, entries=150, sequenceid=145, filesize=11.9 K 2024-11-09T06:17:08,702 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for a2e6477244fddb2a7787090556d9c8bd in 1733ms, sequenceid=145, compaction requested=false 2024-11-09T06:17:08,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:08,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:08,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-09T06:17:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-09T06:17:08,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-09T06:17:08,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1980 sec 2024-11-09T06:17:08,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.2060 sec 2024-11-09T06:17:08,717 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T06:17:08,717 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-09T06:17:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:09,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T06:17:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:09,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109731558e735d74a6f9039164425cb631a_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133027095/Put/seqid=0 2024-11-09T06:17:09,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742212_1388 (size=14794) 2024-11-09T06:17:09,342 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:09,348 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109731558e735d74a6f9039164425cb631a_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109731558e735d74a6f9039164425cb631a_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:09,355 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/bd8bcd5947994ff9b23df1d47f9550b9, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:09,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/bd8bcd5947994ff9b23df1d47f9550b9 is 175, key is test_row_0/A:col10/1731133027095/Put/seqid=0 2024-11-09T06:17:09,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742213_1389 (size=39749) 2024-11-09T06:17:09,383 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/bd8bcd5947994ff9b23df1d47f9550b9 2024-11-09T06:17:09,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/ad53b7438a7c419a8a2e8bcb051e1d7f is 50, key is test_row_0/B:col10/1731133027095/Put/seqid=0 2024-11-09T06:17:09,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742214_1390 (size=12151) 2024-11-09T06:17:09,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/ad53b7438a7c419a8a2e8bcb051e1d7f 2024-11-09T06:17:09,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/0f9d01e01582499ab467ded2ce73714b is 50, key is test_row_0/C:col10/1731133027095/Put/seqid=0 2024-11-09T06:17:09,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742215_1391 (size=12151) 2024-11-09T06:17:09,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133089481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:09,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133089593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:09,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/0f9d01e01582499ab467ded2ce73714b 2024-11-09T06:17:09,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/bd8bcd5947994ff9b23df1d47f9550b9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/bd8bcd5947994ff9b23df1d47f9550b9 2024-11-09T06:17:09,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133089813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:09,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/bd8bcd5947994ff9b23df1d47f9550b9, entries=200, sequenceid=160, filesize=38.8 K 2024-11-09T06:17:09,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/ad53b7438a7c419a8a2e8bcb051e1d7f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/ad53b7438a7c419a8a2e8bcb051e1d7f 2024-11-09T06:17:09,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/ad53b7438a7c419a8a2e8bcb051e1d7f, entries=150, sequenceid=160, filesize=11.9 K 2024-11-09T06:17:09,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/0f9d01e01582499ab467ded2ce73714b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/0f9d01e01582499ab467ded2ce73714b 2024-11-09T06:17:09,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/0f9d01e01582499ab467ded2ce73714b, entries=150, sequenceid=160, filesize=11.9 K 2024-11-09T06:17:09,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a2e6477244fddb2a7787090556d9c8bd in 527ms, sequenceid=160, compaction requested=true 2024-11-09T06:17:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:09,844 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:09,844 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:09,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:09,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:09,845 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:09,846 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/B is initiating minor compaction (all files) 2024-11-09T06:17:09,846 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:09,846 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/B in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:09,846 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/A is initiating minor compaction (all files) 2024-11-09T06:17:09,846 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/A in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:09,846 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/20abcb8b3e12466dadb218106a7557b8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/08027f7e656548e1a20404a15a5e8a17, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/ad53b7438a7c419a8a2e8bcb051e1d7f] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=35.7 K 2024-11-09T06:17:09,846 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/51871e6623ac43569435bddd62e00e40, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7f4876a08df14e288718948ce8f3ec0b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/bd8bcd5947994ff9b23df1d47f9550b9] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=99.6 K 2024-11-09T06:17:09,846 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:09,846 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/51871e6623ac43569435bddd62e00e40, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7f4876a08df14e288718948ce8f3ec0b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/bd8bcd5947994ff9b23df1d47f9550b9] 2024-11-09T06:17:09,846 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 20abcb8b3e12466dadb218106a7557b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731133025136 2024-11-09T06:17:09,846 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51871e6623ac43569435bddd62e00e40, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731133025136 2024-11-09T06:17:09,847 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 08027f7e656548e1a20404a15a5e8a17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731133026425 2024-11-09T06:17:09,847 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f4876a08df14e288718948ce8f3ec0b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731133026425 2024-11-09T06:17:09,847 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting ad53b7438a7c419a8a2e8bcb051e1d7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731133027035 2024-11-09T06:17:09,847 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd8bcd5947994ff9b23df1d47f9550b9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731133027035 2024-11-09T06:17:09,861 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:09,862 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#B#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:09,862 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/3ffd67f0505648e599cc9dfd122b13e1 is 50, key is test_row_0/B:col10/1731133027095/Put/seqid=0 2024-11-09T06:17:09,867 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109b5121f19e47541bfaf80c1d0f4eb2549_a2e6477244fddb2a7787090556d9c8bd store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:09,869 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109b5121f19e47541bfaf80c1d0f4eb2549_a2e6477244fddb2a7787090556d9c8bd, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:09,869 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b5121f19e47541bfaf80c1d0f4eb2549_a2e6477244fddb2a7787090556d9c8bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:09,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742217_1393 (size=4469) 2024-11-09T06:17:09,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742216_1392 (size=12459) 2024-11-09T06:17:09,891 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#A#compaction#333 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:09,892 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/8c64c24af4854507b35c67205b52558c is 175, key is test_row_0/A:col10/1731133027095/Put/seqid=0 2024-11-09T06:17:09,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742218_1394 (size=31413) 2024-11-09T06:17:09,917 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/8c64c24af4854507b35c67205b52558c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/8c64c24af4854507b35c67205b52558c 2024-11-09T06:17:09,924 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/A of a2e6477244fddb2a7787090556d9c8bd into 8c64c24af4854507b35c67205b52558c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:09,924 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:09,924 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/A, priority=13, startTime=1731133029844; duration=0sec 2024-11-09T06:17:09,924 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:09,924 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:A 2024-11-09T06:17:09,924 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:09,925 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:09,925 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/C is initiating minor compaction (all files) 2024-11-09T06:17:09,926 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/C in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:09,926 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/86dce45162f34844ac54dee1258b0525, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/ef01a8ddd40b43548dec1b38fec797d0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/0f9d01e01582499ab467ded2ce73714b] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=35.7 K 2024-11-09T06:17:09,926 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86dce45162f34844ac54dee1258b0525, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731133025136 2024-11-09T06:17:09,926 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef01a8ddd40b43548dec1b38fec797d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731133026425 2024-11-09T06:17:09,927 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f9d01e01582499ab467ded2ce73714b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731133027035 2024-11-09T06:17:09,940 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#C#compaction#334 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:09,941 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/f4bf968b731f49c6952cd4a151cf28f2 is 50, key is test_row_0/C:col10/1731133027095/Put/seqid=0 2024-11-09T06:17:09,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742219_1395 (size=12459) 2024-11-09T06:17:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:10,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-09T06:17:10,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:10,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:10,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:10,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:10,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:10,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:10,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095824b6c374d240d888baeb40ed42c28d_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133030137/Put/seqid=0 2024-11-09T06:17:10,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742220_1396 (size=14794) 2024-11-09T06:17:10,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133090236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:10,298 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/3ffd67f0505648e599cc9dfd122b13e1 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/3ffd67f0505648e599cc9dfd122b13e1 2024-11-09T06:17:10,305 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/B of a2e6477244fddb2a7787090556d9c8bd into 3ffd67f0505648e599cc9dfd122b13e1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:10,305 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:10,305 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/B, priority=13, startTime=1731133029844; duration=0sec 2024-11-09T06:17:10,305 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:10,305 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:B 2024-11-09T06:17:10,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133090353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:10,366 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/f4bf968b731f49c6952cd4a151cf28f2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/f4bf968b731f49c6952cd4a151cf28f2 2024-11-09T06:17:10,370 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/C of a2e6477244fddb2a7787090556d9c8bd into f4bf968b731f49c6952cd4a151cf28f2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:10,370 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:10,370 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/C, priority=13, startTime=1731133029845; duration=0sec 2024-11-09T06:17:10,370 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:10,370 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:C 2024-11-09T06:17:10,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133090572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:10,596 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:10,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095824b6c374d240d888baeb40ed42c28d_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095824b6c374d240d888baeb40ed42c28d_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:10,602 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/b730bc71a7204635a0c9b0abcb279fc5, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:10,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/b730bc71a7204635a0c9b0abcb279fc5 is 175, key is test_row_0/A:col10/1731133030137/Put/seqid=0 2024-11-09T06:17:10,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742221_1397 (size=39749) 2024-11-09T06:17:10,614 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/b730bc71a7204635a0c9b0abcb279fc5 2024-11-09T06:17:10,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/5104a52df88d44978eb3b7a74a119f13 is 50, key is test_row_0/B:col10/1731133030137/Put/seqid=0 2024-11-09T06:17:10,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742222_1398 (size=12151) 2024-11-09T06:17:10,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/5104a52df88d44978eb3b7a74a119f13 2024-11-09T06:17:10,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/b5a1b81dc96a47e28f189fb3bd5e0ffd is 50, key is test_row_0/C:col10/1731133030137/Put/seqid=0 2024-11-09T06:17:10,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133090663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:10,673 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_0' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T06:17:10,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742223_1399 (size=12151) 2024-11-09T06:17:10,684 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:10,684 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-09T06:17:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T06:17:10,687 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:10,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133090674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:10,688 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:10,689 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:10,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133090674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:10,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:10,690 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:10,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:10,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133090684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:10,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/b5a1b81dc96a47e28f189fb3bd5e0ffd 2024-11-09T06:17:10,693 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_0' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:10,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/b730bc71a7204635a0c9b0abcb279fc5 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b730bc71a7204635a0c9b0abcb279fc5 2024-11-09T06:17:10,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b730bc71a7204635a0c9b0abcb279fc5, entries=200, sequenceid=182, filesize=38.8 K 2024-11-09T06:17:10,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/5104a52df88d44978eb3b7a74a119f13 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/5104a52df88d44978eb3b7a74a119f13 2024-11-09T06:17:10,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/5104a52df88d44978eb3b7a74a119f13, entries=150, sequenceid=182, filesize=11.9 K 2024-11-09T06:17:10,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/b5a1b81dc96a47e28f189fb3bd5e0ffd as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/b5a1b81dc96a47e28f189fb3bd5e0ffd 2024-11-09T06:17:10,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/b5a1b81dc96a47e28f189fb3bd5e0ffd, entries=150, sequenceid=182, filesize=11.9 K 2024-11-09T06:17:10,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for a2e6477244fddb2a7787090556d9c8bd in 590ms, sequenceid=182, compaction requested=false 2024-11-09T06:17:10,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T06:17:10,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T06:17:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:10,846 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T06:17:10,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:10,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:10,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:10,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:10,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:10,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:10,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110936f4b782fd6b40a3aa29880de5bc353a_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133030177/Put/seqid=0 2024-11-09T06:17:10,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742224_1400 (size=12304) 2024-11-09T06:17:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:10,875 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110936f4b782fd6b40a3aa29880de5bc353a_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110936f4b782fd6b40a3aa29880de5bc353a_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:10,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/0158bbc8b166486c9336e87422608a1e, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/0158bbc8b166486c9336e87422608a1e is 175, key is test_row_0/A:col10/1731133030177/Put/seqid=0 2024-11-09T06:17:10,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:10,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:10,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742225_1401 (size=31105) 2024-11-09T06:17:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T06:17:11,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133031003, max timestamp=1731133031035Num uniques 859; 2024-11-09T06:17:11,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:17:11,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133091079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:11,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133091193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:11,306 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/0158bbc8b166486c9336e87422608a1e 2024-11-09T06:17:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T06:17:11,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/c2e16a6d5e7549ee82579323b4862d40 is 50, key is test_row_0/B:col10/1731133030177/Put/seqid=0 2024-11-09T06:17:11,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742226_1402 (size=12151) 2024-11-09T06:17:11,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133091402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:11,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:11,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:11,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:11,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133091713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:11,755 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/c2e16a6d5e7549ee82579323b4862d40 2024-11-09T06:17:11,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/08ae04031e09406a900effd26db50dcb is 50, key is test_row_0/C:col10/1731133030177/Put/seqid=0 2024-11-09T06:17:11,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742227_1403 (size=12151) 2024-11-09T06:17:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T06:17:12,223 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/08ae04031e09406a900effd26db50dcb 2024-11-09T06:17:12,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/0158bbc8b166486c9336e87422608a1e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/0158bbc8b166486c9336e87422608a1e 2024-11-09T06:17:12,235 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/0158bbc8b166486c9336e87422608a1e, entries=150, sequenceid=195, filesize=30.4 K 2024-11-09T06:17:12,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/c2e16a6d5e7549ee82579323b4862d40 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/c2e16a6d5e7549ee82579323b4862d40 2024-11-09T06:17:12,240 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/c2e16a6d5e7549ee82579323b4862d40, entries=150, sequenceid=195, filesize=11.9 K 2024-11-09T06:17:12,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/08ae04031e09406a900effd26db50dcb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/08ae04031e09406a900effd26db50dcb 2024-11-09T06:17:12,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133092233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:12,246 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/08ae04031e09406a900effd26db50dcb, entries=150, sequenceid=195, filesize=11.9 K 2024-11-09T06:17:12,247 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for a2e6477244fddb2a7787090556d9c8bd in 1401ms, sequenceid=195, compaction requested=true 2024-11-09T06:17:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-09T06:17:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-09T06:17:12,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-09T06:17:12,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5590 sec 2024-11-09T06:17:12,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.5660 sec 2024-11-09T06:17:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T06:17:12,843 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:12,844 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-09T06:17:12,848 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:12,850 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:12,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T06:17:12,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T06:17:13,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T06:17:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:13,003 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=123.00 KB heapSize=361.72 KB 2024-11-09T06:17:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:13,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:13,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:13,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:13,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093bf9e130ed6047caace734b227856b7c_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133031078/Put/seqid=0 2024-11-09T06:17:13,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742228_1404 (size=12304) 2024-11-09T06:17:13,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T06:17:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:13,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:13,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133093337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:13,437 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093bf9e130ed6047caace734b227856b7c_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093bf9e130ed6047caace734b227856b7c_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:13,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/eb213997cb0745479efa0f41ac36207a, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:13,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/eb213997cb0745479efa0f41ac36207a is 175, key is test_row_0/A:col10/1731133031078/Put/seqid=0 2024-11-09T06:17:13,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742229_1405 (size=31105) 2024-11-09T06:17:13,445 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/eb213997cb0745479efa0f41ac36207a 2024-11-09T06:17:13,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/57031539a4ab4d49afbe978fb333312f is 50, key is test_row_0/B:col10/1731133031078/Put/seqid=0 2024-11-09T06:17:13,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742230_1406 (size=12151) 2024-11-09T06:17:13,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133093453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T06:17:13,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133093673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:13,857 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/57031539a4ab4d49afbe978fb333312f 2024-11-09T06:17:13,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/a52e253b27a7492dadacaa4ab1e60724 is 50, key is test_row_0/C:col10/1731133031078/Put/seqid=0 2024-11-09T06:17:13,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742231_1407 (size=12151) 2024-11-09T06:17:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T06:17:14,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133093992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:14,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:14,270 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/a52e253b27a7492dadacaa4ab1e60724 2024-11-09T06:17:14,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/eb213997cb0745479efa0f41ac36207a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/eb213997cb0745479efa0f41ac36207a 2024-11-09T06:17:14,284 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/eb213997cb0745479efa0f41ac36207a, entries=150, sequenceid=221, filesize=30.4 K 2024-11-09T06:17:14,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/57031539a4ab4d49afbe978fb333312f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/57031539a4ab4d49afbe978fb333312f 2024-11-09T06:17:14,287 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/57031539a4ab4d49afbe978fb333312f, entries=150, sequenceid=221, filesize=11.9 K 2024-11-09T06:17:14,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/a52e253b27a7492dadacaa4ab1e60724 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/a52e253b27a7492dadacaa4ab1e60724 2024-11-09T06:17:14,293 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/a52e253b27a7492dadacaa4ab1e60724, entries=150, sequenceid=221, filesize=11.9 K 2024-11-09T06:17:14,294 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~123.00 KB/125950, heapSize ~361.67 KB/370352, currentSize=53.67 KB/54960 for a2e6477244fddb2a7787090556d9c8bd in 1291ms, sequenceid=221, compaction requested=true 2024-11-09T06:17:14,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:14,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:14,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-09T06:17:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-09T06:17:14,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-09T06:17:14,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4450 sec 2024-11-09T06:17:14,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.4590 sec 2024-11-09T06:17:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:14,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T06:17:14,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:14,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:14,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:14,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:14,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:14,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:14,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109eb090b3f0ab0432d8fab94d8ae8cccd1_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133034516/Put/seqid=0 2024-11-09T06:17:14,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742232_1408 (size=14794) 2024-11-09T06:17:14,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133094689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:14,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731133094713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:14,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57540 deadline: 1731133094713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:14,722 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_0' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8277 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:14,722 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:14,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57544 deadline: 1731133094722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:14,731 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:14,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57536 deadline: 1731133094732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:14,737 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_0' in a2e6477244fddb2a7787090556d9c8bd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8308 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:14,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133094802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:14,938 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:14,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109eb090b3f0ab0432d8fab94d8ae8cccd1_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eb090b3f0ab0432d8fab94d8ae8cccd1_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:14,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7df7630d58a54121b91021f04166f018, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:14,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7df7630d58a54121b91021f04166f018 is 175, key is test_row_0/A:col10/1731133034516/Put/seqid=0 2024-11-09T06:17:14,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742233_1409 (size=39749) 2024-11-09T06:17:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T06:17:14,993 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:14,993 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-09T06:17:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T06:17:14,995 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:14,995 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:14,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:15,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133095013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T06:17:15,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:15,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:15,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:15,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:15,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T06:17:15,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133095332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:15,354 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7df7630d58a54121b91021f04166f018 2024-11-09T06:17:15,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/261453f104d44415a6c15b96173e487b is 50, key is test_row_0/B:col10/1731133034516/Put/seqid=0 2024-11-09T06:17:15,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742234_1410 (size=12151) 2024-11-09T06:17:15,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T06:17:15,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:15,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:15,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/261453f104d44415a6c15b96173e487b 2024-11-09T06:17:15,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/135188a920fb4fa7b567e8b0efbad861 is 50, key is test_row_0/C:col10/1731133034516/Put/seqid=0 2024-11-09T06:17:15,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742235_1411 (size=12151) 2024-11-09T06:17:15,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/135188a920fb4fa7b567e8b0efbad861 2024-11-09T06:17:15,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/7df7630d58a54121b91021f04166f018 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7df7630d58a54121b91021f04166f018 2024-11-09T06:17:15,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7df7630d58a54121b91021f04166f018, entries=200, sequenceid=233, filesize=38.8 K 2024-11-09T06:17:15,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/261453f104d44415a6c15b96173e487b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/261453f104d44415a6c15b96173e487b 2024-11-09T06:17:15,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/261453f104d44415a6c15b96173e487b, entries=150, sequenceid=233, filesize=11.9 K 2024-11-09T06:17:15,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/135188a920fb4fa7b567e8b0efbad861 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/135188a920fb4fa7b567e8b0efbad861 2024-11-09T06:17:15,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/135188a920fb4fa7b567e8b0efbad861, entries=150, sequenceid=233, filesize=11.9 K 2024-11-09T06:17:15,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a2e6477244fddb2a7787090556d9c8bd in 1331ms, sequenceid=233, compaction requested=true 2024-11-09T06:17:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:15,850 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T06:17:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T06:17:15,850 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T06:17:15,853 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 173121 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T06:17:15,853 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/A is initiating minor compaction (all files) 2024-11-09T06:17:15,853 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/A in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,854 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/8c64c24af4854507b35c67205b52558c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b730bc71a7204635a0c9b0abcb279fc5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/0158bbc8b166486c9336e87422608a1e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/eb213997cb0745479efa0f41ac36207a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7df7630d58a54121b91021f04166f018] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=169.1 K 2024-11-09T06:17:15,854 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,854 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/8c64c24af4854507b35c67205b52558c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b730bc71a7204635a0c9b0abcb279fc5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/0158bbc8b166486c9336e87422608a1e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/eb213997cb0745479efa0f41ac36207a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7df7630d58a54121b91021f04166f018] 2024-11-09T06:17:15,854 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T06:17:15,855 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/B is initiating minor compaction (all files) 2024-11-09T06:17:15,855 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/B in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,855 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/3ffd67f0505648e599cc9dfd122b13e1, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/5104a52df88d44978eb3b7a74a119f13, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/c2e16a6d5e7549ee82579323b4862d40, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/57031539a4ab4d49afbe978fb333312f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/261453f104d44415a6c15b96173e487b] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=59.6 K 2024-11-09T06:17:15,856 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ffd67f0505648e599cc9dfd122b13e1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731133027035 2024-11-09T06:17:15,856 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c64c24af4854507b35c67205b52558c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731133027035 2024-11-09T06:17:15,857 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b730bc71a7204635a0c9b0abcb279fc5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731133029446 2024-11-09T06:17:15,857 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5104a52df88d44978eb3b7a74a119f13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731133029446 2024-11-09T06:17:15,858 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting c2e16a6d5e7549ee82579323b4862d40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731133030177 2024-11-09T06:17:15,858 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0158bbc8b166486c9336e87422608a1e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731133030177 2024-11-09T06:17:15,858 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 57031539a4ab4d49afbe978fb333312f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731133031051 2024-11-09T06:17:15,858 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb213997cb0745479efa0f41ac36207a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731133031051 2024-11-09T06:17:15,861 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 261453f104d44415a6c15b96173e487b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731133033311 2024-11-09T06:17:15,861 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7df7630d58a54121b91021f04166f018, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731133033298 2024-11-09T06:17:15,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133034643, max timestamp=1731133035863Num uniques 859; 2024-11-09T06:17:15,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:17:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:15,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133034643, max timestamp=1731133035863Num uniques 859; 2024-11-09T06:17:15,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:17:15,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-09T06:17:15,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133034643, max timestamp=1731133035863Num uniques 859; 2024-11-09T06:17:15,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:17:15,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:15,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:15,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:15,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:15,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:15,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:15,894 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:15,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:15,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:15,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:15,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:15,932 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#B#compaction#348 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:15,933 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/c2acce878ebe491ebdc0d63b43302440 is 50, key is test_row_0/B:col10/1731133034516/Put/seqid=0 2024-11-09T06:17:15,941 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109f8e15bc4a03549b492743462459c4307_a2e6477244fddb2a7787090556d9c8bd store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:15,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093a229450f0dc4faf98ad6da74f96f81b_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133034663/Put/seqid=0 2024-11-09T06:17:15,948 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109f8e15bc4a03549b492743462459c4307_a2e6477244fddb2a7787090556d9c8bd, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:15,950 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f8e15bc4a03549b492743462459c4307_a2e6477244fddb2a7787090556d9c8bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:15,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742236_1412 (size=12629) 2024-11-09T06:17:15,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742237_1413 (size=12304) 2024-11-09T06:17:16,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742238_1414 (size=4469) 2024-11-09T06:17:16,005 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#A#compaction#347 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:16,006 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/95ffb0bf5e9b4de1bdd3c8a865cae6d9 is 175, key is test_row_0/A:col10/1731133034516/Put/seqid=0 2024-11-09T06:17:16,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742239_1415 (size=31583) 2024-11-09T06:17:16,034 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/95ffb0bf5e9b4de1bdd3c8a865cae6d9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95ffb0bf5e9b4de1bdd3c8a865cae6d9 2024-11-09T06:17:16,051 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/A of a2e6477244fddb2a7787090556d9c8bd into 95ffb0bf5e9b4de1bdd3c8a865cae6d9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:16,051 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:16,051 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/A, priority=11, startTime=1731133035850; duration=0sec 2024-11-09T06:17:16,052 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:16,052 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:A 2024-11-09T06:17:16,052 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T06:17:16,054 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T06:17:16,055 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/C is initiating minor compaction (all files) 2024-11-09T06:17:16,055 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/C in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,055 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/f4bf968b731f49c6952cd4a151cf28f2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/b5a1b81dc96a47e28f189fb3bd5e0ffd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/08ae04031e09406a900effd26db50dcb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/a52e253b27a7492dadacaa4ab1e60724, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/135188a920fb4fa7b567e8b0efbad861] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=59.6 K 2024-11-09T06:17:16,055 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4bf968b731f49c6952cd4a151cf28f2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731133027035 2024-11-09T06:17:16,056 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5a1b81dc96a47e28f189fb3bd5e0ffd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731133029446 2024-11-09T06:17:16,056 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08ae04031e09406a900effd26db50dcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731133030177 2024-11-09T06:17:16,056 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a52e253b27a7492dadacaa4ab1e60724, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731133031051 2024-11-09T06:17:16,057 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 135188a920fb4fa7b567e8b0efbad861, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731133033311 2024-11-09T06:17:16,075 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:16,076 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/dd93bfac476c413fab5fc982004e40d4 is 50, key is test_row_0/C:col10/1731133034516/Put/seqid=0 2024-11-09T06:17:16,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:16,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:16,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742240_1416 (size=12629) 2024-11-09T06:17:16,133 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/dd93bfac476c413fab5fc982004e40d4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/dd93bfac476c413fab5fc982004e40d4 2024-11-09T06:17:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T06:17:16,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133036093, max timestamp=1731133036132Num uniques 859; 2024-11-09T06:17:16,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:16,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133036093, max timestamp=1731133036132Num uniques 859; 2024-11-09T06:17:16,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:16,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133036093, max timestamp=1731133036132Num uniques 859; 2024-11-09T06:17:16,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:16,148 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/C of a2e6477244fddb2a7787090556d9c8bd into dd93bfac476c413fab5fc982004e40d4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:16,148 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:16,148 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/C, priority=11, startTime=1731133035850; duration=0sec 2024-11-09T06:17:16,148 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:16,149 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:C 2024-11-09T06:17:16,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:16,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:16,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731133036283, max timestamp=1731133036329Num uniques 1068; 2024-11-09T06:17:16,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731133036283, max timestamp=1731133036329Num uniques 1068; 2024-11-09T06:17:16,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731133036283, max timestamp=1731133036329Num uniques 1068; 2024-11-09T06:17:16,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T06:17:16,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T06:17:16,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T06:17:16,372 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/c2acce878ebe491ebdc0d63b43302440 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/c2acce878ebe491ebdc0d63b43302440 2024-11-09T06:17:16,376 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/B of a2e6477244fddb2a7787090556d9c8bd into c2acce878ebe491ebdc0d63b43302440(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:16,376 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:16,376 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/B, priority=11, startTime=1731133035850; duration=0sec 2024-11-09T06:17:16,376 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:16,376 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:B 2024-11-09T06:17:16,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:16,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:16,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,399 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:16,408 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093a229450f0dc4faf98ad6da74f96f81b_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093a229450f0dc4faf98ad6da74f96f81b_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:16,413 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/10e4791680ed42b69d1f682a547cbed4, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:16,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/10e4791680ed42b69d1f682a547cbed4 is 175, key is test_row_0/A:col10/1731133034663/Put/seqid=0 2024-11-09T06:17:16,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742241_1417 (size=31105) 2024-11-09T06:17:16,426 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/10e4791680ed42b69d1f682a547cbed4 2024-11-09T06:17:16,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/8b59568f498c4f108f6b484287b5337d is 50, key is test_row_0/B:col10/1731133034663/Put/seqid=0 2024-11-09T06:17:16,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742242_1418 (size=12151) 2024-11-09T06:17:16,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731133036407, max timestamp=1731133036519Num uniques 1027; 2024-11-09T06:17:16,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731133036407, max timestamp=1731133036519Num uniques 1027; 2024-11-09T06:17:16,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-09T06:17:16,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-09T06:17:16,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731133036407, max timestamp=1731133036519Num uniques 1027; 2024-11-09T06:17:16,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-09T06:17:16,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:16,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:16,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731133036613, max timestamp=1731133036714Num uniques 1036; 2024-11-09T06:17:16,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731133036613, max timestamp=1731133036714Num uniques 1036; 2024-11-09T06:17:16,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-09T06:17:16,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-09T06:17:16,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731133036613, max timestamp=1731133036714Num uniques 1036; 2024-11-09T06:17:16,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-09T06:17:16,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/8b59568f498c4f108f6b484287b5337d 2024-11-09T06:17:16,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/8dd4be4beb074b478389fdc68633e0e1 is 50, key is test_row_0/C:col10/1731133034663/Put/seqid=0 2024-11-09T06:17:16,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:16,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:16,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742243_1419 (size=12151) 2024-11-09T06:17:16,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731133036861, max timestamp=1731133036894Num uniques 1095; 2024-11-09T06:17:16,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-09T06:17:16,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731133036861, max timestamp=1731133036894Num uniques 1095; 2024-11-09T06:17:16,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-09T06:17:16,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731133036861, max timestamp=1731133036894Num uniques 1095; 2024-11-09T06:17:16,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-09T06:17:17,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:17,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:17,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:17,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:17,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:17,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:17,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731133037043, max timestamp=1731133037077Num uniques 1054; 2024-11-09T06:17:17,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-09T06:17:17,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731133037043, max timestamp=1731133037077Num uniques 1054; 2024-11-09T06:17:17,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-09T06:17:17,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731133037043, max timestamp=1731133037077Num uniques 1054; 2024-11-09T06:17:17,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-09T06:17:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T06:17:17,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:17,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:17,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:17,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:17,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:17,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:17,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731133037200, max timestamp=1731133037269Num uniques 1013; 2024-11-09T06:17:17,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731133037200, max timestamp=1731133037269Num uniques 1013; 2024-11-09T06:17:17,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-09T06:17:17,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-09T06:17:17,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/8dd4be4beb074b478389fdc68633e0e1 2024-11-09T06:17:17,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731133037200, max timestamp=1731133037269Num uniques 1013; 2024-11-09T06:17:17,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-09T06:17:17,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/10e4791680ed42b69d1f682a547cbed4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/10e4791680ed42b69d1f682a547cbed4 2024-11-09T06:17:17,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/10e4791680ed42b69d1f682a547cbed4, entries=150, sequenceid=254, filesize=30.4 K 2024-11-09T06:17:17,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/8b59568f498c4f108f6b484287b5337d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8b59568f498c4f108f6b484287b5337d 2024-11-09T06:17:17,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8b59568f498c4f108f6b484287b5337d, entries=150, sequenceid=254, filesize=11.9 K 2024-11-09T06:17:17,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/8dd4be4beb074b478389fdc68633e0e1 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/8dd4be4beb074b478389fdc68633e0e1 2024-11-09T06:17:17,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/8dd4be4beb074b478389fdc68633e0e1, entries=150, sequenceid=254, filesize=11.9 K 2024-11-09T06:17:17,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=60.38 KB/61830 for a2e6477244fddb2a7787090556d9c8bd in 1439ms, sequenceid=254, compaction requested=false 2024-11-09T06:17:17,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:17,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T06:17:17,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:17,318 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=60.38 KB heapSize=160.78 KB 2024-11-09T06:17:17,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:17,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:17,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:17,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:17,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:17,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:17,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110943c06bb04bb645c2b533134d43331a20_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133037266/Put/seqid=0 2024-11-09T06:17:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:17,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:17,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742244_1420 (size=14994) 2024-11-09T06:17:17,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:17,375 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110943c06bb04bb645c2b533134d43331a20_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110943c06bb04bb645c2b533134d43331a20_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:17,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/18b501dae36e4b088aa874e8fb61efc9, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:17,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/18b501dae36e4b088aa874e8fb61efc9 is 175, key is test_row_0/A:col10/1731133037266/Put/seqid=0 2024-11-09T06:17:17,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742245_1421 (size=39949) 2024-11-09T06:17:17,416 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=385, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/18b501dae36e4b088aa874e8fb61efc9 2024-11-09T06:17:17,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/8608e84c56384560a41379735b677e75 is 50, key is test_row_0/B:col10/1731133037266/Put/seqid=0 2024-11-09T06:17:17,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742246_1422 (size=12301) 2024-11-09T06:17:17,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133037475, max timestamp=1731133037549Num uniques 859; 2024-11-09T06:17:17,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133037475, max timestamp=1731133037549Num uniques 859; 2024-11-09T06:17:17,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133037475, max timestamp=1731133037549Num uniques 859; 2024-11-09T06:17:17,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:17,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:17,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:17,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133097770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:17,867 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/8608e84c56384560a41379735b677e75 2024-11-09T06:17:17,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/12e5db32869e428ab94fe02606c256ed is 50, key is test_row_0/C:col10/1731133037266/Put/seqid=0 2024-11-09T06:17:17,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133097893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:17,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742247_1423 (size=12301) 2024-11-09T06:17:17,918 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/12e5db32869e428ab94fe02606c256ed 2024-11-09T06:17:17,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/18b501dae36e4b088aa874e8fb61efc9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/18b501dae36e4b088aa874e8fb61efc9 2024-11-09T06:17:17,928 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/18b501dae36e4b088aa874e8fb61efc9, entries=200, sequenceid=385, filesize=39.0 K 2024-11-09T06:17:17,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/8608e84c56384560a41379735b677e75 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8608e84c56384560a41379735b677e75 2024-11-09T06:17:17,934 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8608e84c56384560a41379735b677e75, entries=150, sequenceid=385, filesize=12.0 K 2024-11-09T06:17:17,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/12e5db32869e428ab94fe02606c256ed as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/12e5db32869e428ab94fe02606c256ed 2024-11-09T06:17:17,945 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/12e5db32869e428ab94fe02606c256ed, entries=150, sequenceid=385, filesize=12.0 K 2024-11-09T06:17:17,946 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~160.73 KB/164592, currentSize=127.47 KB/130530 for a2e6477244fddb2a7787090556d9c8bd in 629ms, sequenceid=385, compaction requested=true 2024-11-09T06:17:17,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:17,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:17,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-09T06:17:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-09T06:17:17,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-09T06:17:17,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9530 sec 2024-11-09T06:17:17,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.9620 sec 2024-11-09T06:17:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:18,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-09T06:17:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:18,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:18,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110944f508b97af0473f8db390f0c4cf38d0_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133037744/Put/seqid=0 2024-11-09T06:17:18,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742248_1424 (size=14994) 2024-11-09T06:17:18,189 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:18,196 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110944f508b97af0473f8db390f0c4cf38d0_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110944f508b97af0473f8db390f0c4cf38d0_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:18,197 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/91c123b9a3bb4ee09d77c485a74a7fa4, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:18,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/91c123b9a3bb4ee09d77c485a74a7fa4 is 175, key is test_row_0/A:col10/1731133037744/Put/seqid=0 2024-11-09T06:17:18,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742249_1425 (size=39949) 2024-11-09T06:17:18,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133098224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:18,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:18,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:18,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133098343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:18,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133098563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:18,611 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/91c123b9a3bb4ee09d77c485a74a7fa4 2024-11-09T06:17:18,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/03eba2d63e7644bb8a211e974ba88f58 is 50, key is test_row_0/B:col10/1731133037744/Put/seqid=0 2024-11-09T06:17:18,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742250_1426 (size=12301) 2024-11-09T06:17:18,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/03eba2d63e7644bb8a211e974ba88f58 2024-11-09T06:17:18,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/bb033ea6f9bc43aca54924e831fd102a is 50, key is test_row_0/C:col10/1731133037744/Put/seqid=0 2024-11-09T06:17:18,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742251_1427 (size=12301) 2024-11-09T06:17:18,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133098883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:19,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/bb033ea6f9bc43aca54924e831fd102a 2024-11-09T06:17:19,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/91c123b9a3bb4ee09d77c485a74a7fa4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/91c123b9a3bb4ee09d77c485a74a7fa4 2024-11-09T06:17:19,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/91c123b9a3bb4ee09d77c485a74a7fa4, entries=200, sequenceid=421, filesize=39.0 K 2024-11-09T06:17:19,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/03eba2d63e7644bb8a211e974ba88f58 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/03eba2d63e7644bb8a211e974ba88f58 2024-11-09T06:17:19,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/03eba2d63e7644bb8a211e974ba88f58, entries=150, sequenceid=421, filesize=12.0 K 2024-11-09T06:17:19,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/bb033ea6f9bc43aca54924e831fd102a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/bb033ea6f9bc43aca54924e831fd102a 2024-11-09T06:17:19,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/bb033ea6f9bc43aca54924e831fd102a, entries=150, sequenceid=421, filesize=12.0 K 2024-11-09T06:17:19,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a2e6477244fddb2a7787090556d9c8bd in 1010ms, sequenceid=421, compaction requested=true 2024-11-09T06:17:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:19,135 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:19,135 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:19,138 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:19,138 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/B is initiating minor compaction (all files) 2024-11-09T06:17:19,138 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/B in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:19,138 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/c2acce878ebe491ebdc0d63b43302440, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8b59568f498c4f108f6b484287b5337d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8608e84c56384560a41379735b677e75, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/03eba2d63e7644bb8a211e974ba88f58] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=48.2 K 2024-11-09T06:17:19,138 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142586 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:19,138 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/A is initiating minor compaction (all files) 2024-11-09T06:17:19,138 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/A in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:19,139 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95ffb0bf5e9b4de1bdd3c8a865cae6d9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/10e4791680ed42b69d1f682a547cbed4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/18b501dae36e4b088aa874e8fb61efc9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/91c123b9a3bb4ee09d77c485a74a7fa4] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=139.2 K 2024-11-09T06:17:19,139 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:19,139 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95ffb0bf5e9b4de1bdd3c8a865cae6d9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/10e4791680ed42b69d1f682a547cbed4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/18b501dae36e4b088aa874e8fb61efc9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/91c123b9a3bb4ee09d77c485a74a7fa4] 2024-11-09T06:17:19,139 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting c2acce878ebe491ebdc0d63b43302440, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731133033311 2024-11-09T06:17:19,139 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95ffb0bf5e9b4de1bdd3c8a865cae6d9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731133033311 2024-11-09T06:17:19,140 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b59568f498c4f108f6b484287b5337d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731133034663 2024-11-09T06:17:19,140 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10e4791680ed42b69d1f682a547cbed4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731133034663 2024-11-09T06:17:19,140 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8608e84c56384560a41379735b677e75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731133037266 2024-11-09T06:17:19,140 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18b501dae36e4b088aa874e8fb61efc9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731133037266 2024-11-09T06:17:19,141 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 03eba2d63e7644bb8a211e974ba88f58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731133037706 2024-11-09T06:17:19,141 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91c123b9a3bb4ee09d77c485a74a7fa4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731133037706 2024-11-09T06:17:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T06:17:19,153 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:19,153 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:19,153 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-09T06:17:19,155 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T06:17:19,156 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:19,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:19,156 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110910008ec90f5240f59fc83dce6199387d_a2e6477244fddb2a7787090556d9c8bd store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:19,161 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110910008ec90f5240f59fc83dce6199387d_a2e6477244fddb2a7787090556d9c8bd, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:19,161 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110910008ec90f5240f59fc83dce6199387d_a2e6477244fddb2a7787090556d9c8bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:19,173 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#B#compaction#360 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:19,174 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/bd9164e992494caa82d769a5788793f3 is 50, key is test_row_0/B:col10/1731133037744/Put/seqid=0 2024-11-09T06:17:19,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742252_1428 (size=4469) 2024-11-09T06:17:19,189 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#A#compaction#359 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:19,189 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/b307b2f0c38a40a89f0f2bc7589978f6 is 175, key is test_row_0/A:col10/1731133037744/Put/seqid=0 2024-11-09T06:17:19,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742253_1429 (size=12915) 2024-11-09T06:17:19,209 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/bd9164e992494caa82d769a5788793f3 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/bd9164e992494caa82d769a5788793f3 2024-11-09T06:17:19,214 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/B of a2e6477244fddb2a7787090556d9c8bd into bd9164e992494caa82d769a5788793f3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:19,215 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:19,215 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/B, priority=12, startTime=1731133039135; duration=0sec 2024-11-09T06:17:19,215 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:19,215 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:B 2024-11-09T06:17:19,215 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:19,218 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:19,218 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/C is initiating minor compaction (all files) 2024-11-09T06:17:19,218 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/C in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:19,219 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/dd93bfac476c413fab5fc982004e40d4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/8dd4be4beb074b478389fdc68633e0e1, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/12e5db32869e428ab94fe02606c256ed, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/bb033ea6f9bc43aca54924e831fd102a] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=48.2 K 2024-11-09T06:17:19,219 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting dd93bfac476c413fab5fc982004e40d4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731133033311 2024-11-09T06:17:19,219 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dd4be4beb074b478389fdc68633e0e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731133034663 2024-11-09T06:17:19,220 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e5db32869e428ab94fe02606c256ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731133037266 2024-11-09T06:17:19,221 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting bb033ea6f9bc43aca54924e831fd102a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731133037706 2024-11-09T06:17:19,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742254_1430 (size=31869) 2024-11-09T06:17:19,234 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#C#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:19,235 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/f2159634dcbc49e1b8ca8043bdda1084 is 50, key is test_row_0/C:col10/1731133037744/Put/seqid=0 2024-11-09T06:17:19,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742255_1431 (size=12915) 2024-11-09T06:17:19,256 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/f2159634dcbc49e1b8ca8043bdda1084 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/f2159634dcbc49e1b8ca8043bdda1084 2024-11-09T06:17:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T06:17:19,264 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/C of a2e6477244fddb2a7787090556d9c8bd into f2159634dcbc49e1b8ca8043bdda1084(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:19,264 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:19,264 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/C, priority=12, startTime=1731133039135; duration=0sec 2024-11-09T06:17:19,264 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:19,265 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:C 2024-11-09T06:17:19,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T06:17:19,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:19,309 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:17:19,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:19,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:19,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:19,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:19,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:19,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a3af045ce41541dc925859f27c9445c7_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133038222/Put/seqid=0 2024-11-09T06:17:19,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742256_1432 (size=12454) 2024-11-09T06:17:19,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:19,346 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a3af045ce41541dc925859f27c9445c7_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a3af045ce41541dc925859f27c9445c7_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:19,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/95b46fc8857d43c7999f4dbb9ae3116b, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:19,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/95b46fc8857d43c7999f4dbb9ae3116b is 175, key is test_row_0/A:col10/1731133038222/Put/seqid=0 2024-11-09T06:17:19,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742257_1433 (size=31255) 2024-11-09T06:17:19,357 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/95b46fc8857d43c7999f4dbb9ae3116b 2024-11-09T06:17:19,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/69a90ef1c06b40759256c9d9eb682b3d is 50, key is test_row_0/B:col10/1731133038222/Put/seqid=0 2024-11-09T06:17:19,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742258_1434 (size=12301) 2024-11-09T06:17:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:19,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T06:17:19,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133039490, max timestamp=1731133039538Num uniques 859; 2024-11-09T06:17:19,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133039490, max timestamp=1731133039538Num uniques 859; 2024-11-09T06:17:19,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:19,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:19,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133039490, max timestamp=1731133039538Num uniques 859; 2024-11-09T06:17:19,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:19,632 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/b307b2f0c38a40a89f0f2bc7589978f6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b307b2f0c38a40a89f0f2bc7589978f6 2024-11-09T06:17:19,638 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/A of a2e6477244fddb2a7787090556d9c8bd into b307b2f0c38a40a89f0f2bc7589978f6(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:19,638 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:19,638 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/A, priority=12, startTime=1731133039135; duration=0sec 2024-11-09T06:17:19,638 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:19,638 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:A 2024-11-09T06:17:19,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133099700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:19,774 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/69a90ef1c06b40759256c9d9eb682b3d 2024-11-09T06:17:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T06:17:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/60bed4e31b374cad90c1842d67922bbe is 50, key is test_row_0/C:col10/1731133038222/Put/seqid=0 2024-11-09T06:17:19,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742259_1435 (size=12301) 2024-11-09T06:17:19,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 354 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133099812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:20,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 355 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133100023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:20,206 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/60bed4e31b374cad90c1842d67922bbe 2024-11-09T06:17:20,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/95b46fc8857d43c7999f4dbb9ae3116b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95b46fc8857d43c7999f4dbb9ae3116b 2024-11-09T06:17:20,218 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95b46fc8857d43c7999f4dbb9ae3116b, entries=150, sequenceid=433, filesize=30.5 K 2024-11-09T06:17:20,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/69a90ef1c06b40759256c9d9eb682b3d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/69a90ef1c06b40759256c9d9eb682b3d 2024-11-09T06:17:20,223 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/69a90ef1c06b40759256c9d9eb682b3d, entries=150, sequenceid=433, filesize=12.0 K 2024-11-09T06:17:20,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/60bed4e31b374cad90c1842d67922bbe as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/60bed4e31b374cad90c1842d67922bbe 2024-11-09T06:17:20,229 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/60bed4e31b374cad90c1842d67922bbe, entries=150, sequenceid=433, filesize=12.0 K 2024-11-09T06:17:20,230 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a2e6477244fddb2a7787090556d9c8bd in 920ms, sequenceid=433, compaction requested=false 2024-11-09T06:17:20,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:20,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:20,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-09T06:17:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-09T06:17:20,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-09T06:17:20,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0750 sec 2024-11-09T06:17:20,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.0810 sec 2024-11-09T06:17:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T06:17:20,293 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:20,293 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-09T06:17:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T06:17:20,295 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:20,296 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:20,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:20,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-09T06:17:20,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:20,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:20,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:20,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:20,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:20,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:20,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110991094308e9514459a72d9a4c17a95898_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133040337/Put/seqid=0 2024-11-09T06:17:20,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742260_1436 (size=14994) 2024-11-09T06:17:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T06:17:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 363 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133100391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:20,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T06:17:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:20,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 364 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133100513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:20,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:20,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:20,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:20,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T06:17:20,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:20,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:20,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:20,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T06:17:20,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 365 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133100722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:20,749 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:20,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T06:17:20,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:20,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:20,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:20,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110991094308e9514459a72d9a4c17a95898_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110991094308e9514459a72d9a4c17a95898_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:20,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,755 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/dcd72c1ab92140bfb3f7095f80fe7ebf, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/dcd72c1ab92140bfb3f7095f80fe7ebf is 175, key is test_row_0/A:col10/1731133040337/Put/seqid=0 2024-11-09T06:17:20,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742261_1437 (size=39949) 2024-11-09T06:17:20,761 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=471, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/dcd72c1ab92140bfb3f7095f80fe7ebf 2024-11-09T06:17:20,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/e07a03c01ff24cd2a198024a4167cbe6 is 50, key is test_row_0/B:col10/1731133040337/Put/seqid=0 2024-11-09T06:17:20,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742262_1438 (size=12301) 2024-11-09T06:17:20,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T06:17:20,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:20,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:20,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:20,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T06:17:21,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 366 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133101042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:21,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T06:17:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:21,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/e07a03c01ff24cd2a198024a4167cbe6 2024-11-09T06:17:21,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/5c7485ddd323468ca78c2f8a93af5c0c is 50, key is test_row_0/C:col10/1731133040337/Put/seqid=0 2024-11-09T06:17:21,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742263_1439 (size=12301) 2024-11-09T06:17:21,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T06:17:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T06:17:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T06:17:21,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T06:17:21,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:21,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:21,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 367 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57518 deadline: 1731133101562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:21,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/5c7485ddd323468ca78c2f8a93af5c0c 2024-11-09T06:17:21,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/dcd72c1ab92140bfb3f7095f80fe7ebf as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/dcd72c1ab92140bfb3f7095f80fe7ebf 2024-11-09T06:17:21,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/dcd72c1ab92140bfb3f7095f80fe7ebf, entries=200, sequenceid=471, filesize=39.0 K 2024-11-09T06:17:21,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/e07a03c01ff24cd2a198024a4167cbe6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/e07a03c01ff24cd2a198024a4167cbe6 2024-11-09T06:17:21,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/e07a03c01ff24cd2a198024a4167cbe6, entries=150, sequenceid=471, filesize=12.0 K 2024-11-09T06:17:21,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/5c7485ddd323468ca78c2f8a93af5c0c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/5c7485ddd323468ca78c2f8a93af5c0c 2024-11-09T06:17:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/5c7485ddd323468ca78c2f8a93af5c0c, entries=150, sequenceid=471, filesize=12.0 K 2024-11-09T06:17:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for a2e6477244fddb2a7787090556d9c8bd in 1273ms, sequenceid=471, compaction requested=true 2024-11-09T06:17:21,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:21,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:21,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:21,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:21,612 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:21,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2e6477244fddb2a7787090556d9c8bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:21,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:21,612 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:21,612 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:21,612 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:21,612 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/A is initiating minor compaction (all files) 2024-11-09T06:17:21,612 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/B is initiating minor compaction (all files) 2024-11-09T06:17:21,612 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/A in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,612 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/B in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,613 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b307b2f0c38a40a89f0f2bc7589978f6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95b46fc8857d43c7999f4dbb9ae3116b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/dcd72c1ab92140bfb3f7095f80fe7ebf] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=100.7 K 2024-11-09T06:17:21,613 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/bd9164e992494caa82d769a5788793f3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/69a90ef1c06b40759256c9d9eb682b3d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/e07a03c01ff24cd2a198024a4167cbe6] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=36.6 K 2024-11-09T06:17:21,613 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,613 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b307b2f0c38a40a89f0f2bc7589978f6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95b46fc8857d43c7999f4dbb9ae3116b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/dcd72c1ab92140bfb3f7095f80fe7ebf] 2024-11-09T06:17:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,613 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b307b2f0c38a40a89f0f2bc7589978f6, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731133037706 2024-11-09T06:17:21,613 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting bd9164e992494caa82d769a5788793f3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731133037706 2024-11-09T06:17:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,614 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95b46fc8857d43c7999f4dbb9ae3116b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731133038160 2024-11-09T06:17:21,614 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 69a90ef1c06b40759256c9d9eb682b3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731133038160 2024-11-09T06:17:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,614 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting e07a03c01ff24cd2a198024a4167cbe6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731133039688 2024-11-09T06:17:21,614 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcd72c1ab92140bfb3f7095f80fe7ebf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731133039669 2024-11-09T06:17:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,621 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#B#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,622 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/7c048c5b1e404bcdb505f4988edee682 is 50, key is test_row_0/B:col10/1731133040337/Put/seqid=0 2024-11-09T06:17:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,625 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742264_1440 (size=13017) 2024-11-09T06:17:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,627 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110936d45db1445944028525b9fd03cf0f81_a2e6477244fddb2a7787090556d9c8bd store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,630 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110936d45db1445944028525b9fd03cf0f81_a2e6477244fddb2a7787090556d9c8bd, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,630 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110936d45db1445944028525b9fd03cf0f81_a2e6477244fddb2a7787090556d9c8bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,637 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/7c048c5b1e404bcdb505f4988edee682 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/7c048c5b1e404bcdb505f4988edee682 2024-11-09T06:17:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742265_1441 (size=4469) 2024-11-09T06:17:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,641 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#A#compaction#369 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,642 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/3ae012b322af4cd79f0e8f4d32881658 is 175, key is test_row_0/A:col10/1731133040337/Put/seqid=0 2024-11-09T06:17:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,645 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/B of a2e6477244fddb2a7787090556d9c8bd into 7c048c5b1e404bcdb505f4988edee682(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:21,645 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:21,645 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/B, priority=13, startTime=1731133041611; duration=0sec 2024-11-09T06:17:21,645 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:21,645 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:B 2024-11-09T06:17:21,645 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,651 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:21,651 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): a2e6477244fddb2a7787090556d9c8bd/C is initiating minor compaction (all files) 2024-11-09T06:17:21,651 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2e6477244fddb2a7787090556d9c8bd/C in TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,651 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/f2159634dcbc49e1b8ca8043bdda1084, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/60bed4e31b374cad90c1842d67922bbe, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/5c7485ddd323468ca78c2f8a93af5c0c] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp, totalSize=36.6 K 2024-11-09T06:17:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,651 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting f2159634dcbc49e1b8ca8043bdda1084, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731133037706 2024-11-09T06:17:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,651 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 60bed4e31b374cad90c1842d67922bbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731133038160 2024-11-09T06:17:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,652 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c7485ddd323468ca78c2f8a93af5c0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731133039688 2024-11-09T06:17:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742266_1442 (size=31971) 2024-11-09T06:17:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,666 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2e6477244fddb2a7787090556d9c8bd#C#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,667 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/9ee4de50fb9b427997983a3e7842965d is 50, key is test_row_0/C:col10/1731133040337/Put/seqid=0 2024-11-09T06:17:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T06:17:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:21,680 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-09T06:17:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742267_1443 (size=13017) 2024-11-09T06:17:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,697 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/9ee4de50fb9b427997983a3e7842965d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/9ee4de50fb9b427997983a3e7842965d 2024-11-09T06:17:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f0cb17415db940d58594517643673664_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133040387/Put/seqid=0 2024-11-09T06:17:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,706 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/C of a2e6477244fddb2a7787090556d9c8bd into 9ee4de50fb9b427997983a3e7842965d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:21,706 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,706 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/C, priority=13, startTime=1731133041612; duration=0sec 2024-11-09T06:17:21,706 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:21,706 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:C 2024-11-09T06:17:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742268_1444 (size=9914) 2024-11-09T06:17:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,067 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/3ae012b322af4cd79f0e8f4d32881658 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/3ae012b322af4cd79f0e8f4d32881658 2024-11-09T06:17:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,073 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2e6477244fddb2a7787090556d9c8bd/A of a2e6477244fddb2a7787090556d9c8bd into 3ae012b322af4cd79f0e8f4d32881658(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:22,073 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:22,073 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., storeName=a2e6477244fddb2a7787090556d9c8bd/A, priority=13, startTime=1731133041611; duration=0sec 2024-11-09T06:17:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,073 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:22,073 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2e6477244fddb2a7787090556d9c8bd:A 2024-11-09T06:17:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,127 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f0cb17415db940d58594517643673664_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f0cb17415db940d58594517643673664_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/10382159871a4cda8155d00d608f02c5, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/10382159871a4cda8155d00d608f02c5 is 175, key is test_row_0/A:col10/1731133040387/Put/seqid=0 2024-11-09T06:17:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742269_1445 (size=22561) 2024-11-09T06:17:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,353 INFO [Thread-1620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1620. 2024-11-09T06:17:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,353 DEBUG [Thread-1620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:22,353 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,354 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,363 INFO [Thread-1626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1626. 2024-11-09T06:17:22,363 DEBUG [Thread-1626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:22,363 DEBUG [Thread-1626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,363 DEBUG [Thread-1626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,374 INFO [Thread-1624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1624. 2024-11-09T06:17:22,374 DEBUG [Thread-1624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:22,374 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,374 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,411 INFO [Thread-1618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1618. 2024-11-09T06:17:22,411 DEBUG [Thread-1618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:22,411 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,411 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:22,421 INFO [Thread-1622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1622. 2024-11-09T06:17:22,421 DEBUG [Thread-1622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:22,421 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,422 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T06:17:22,541 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=481, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/10382159871a4cda8155d00d608f02c5 2024-11-09T06:17:22,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/fe52308e439f43838de78061707a7e5b is 50, key is test_row_0/B:col10/1731133040387/Put/seqid=0 2024-11-09T06:17:22,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742270_1446 (size=9857) 2024-11-09T06:17:22,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. as already flushing 2024-11-09T06:17:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:22,585 INFO [Thread-1607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1607. 2024-11-09T06:17:22,585 DEBUG [Thread-1607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:22,585 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,585 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:22,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:22,957 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/fe52308e439f43838de78061707a7e5b 2024-11-09T06:17:22,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/56a39cc60bdb4645a1e11d52c0115844 is 50, key is test_row_0/C:col10/1731133040387/Put/seqid=0 2024-11-09T06:17:22,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742271_1447 (size=9857) 2024-11-09T06:17:23,366 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/56a39cc60bdb4645a1e11d52c0115844 2024-11-09T06:17:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/10382159871a4cda8155d00d608f02c5 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/10382159871a4cda8155d00d608f02c5 2024-11-09T06:17:23,372 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/10382159871a4cda8155d00d608f02c5, entries=100, sequenceid=481, filesize=22.0 K 2024-11-09T06:17:23,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/fe52308e439f43838de78061707a7e5b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/fe52308e439f43838de78061707a7e5b 2024-11-09T06:17:23,376 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/fe52308e439f43838de78061707a7e5b, entries=100, sequenceid=481, filesize=9.6 K 2024-11-09T06:17:23,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/56a39cc60bdb4645a1e11d52c0115844 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/56a39cc60bdb4645a1e11d52c0115844 2024-11-09T06:17:23,381 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/56a39cc60bdb4645a1e11d52c0115844, entries=100, sequenceid=481, filesize=9.6 K 2024-11-09T06:17:23,381 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=6.71 KB/6870 for a2e6477244fddb2a7787090556d9c8bd in 1701ms, sequenceid=481, compaction requested=false 2024-11-09T06:17:23,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a2e6477244fddb2a7787090556d9c8bd: 2024-11-09T06:17:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-09T06:17:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-09T06:17:23,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-09T06:17:23,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0870 sec 2024-11-09T06:17:23,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.0920 sec 2024-11-09T06:17:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T06:17:24,464 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:24,745 INFO [Thread-1615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1615. 2024-11-09T06:17:24,745 DEBUG [Thread-1615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:24,745 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:24,746 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:24,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:24,766 INFO [Thread-1611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1611. 2024-11-09T06:17:24,766 DEBUG [Thread-1611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:24,766 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:24,766 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:24,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:24,795 INFO [Thread-1609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1609. 2024-11-09T06:17:24,795 DEBUG [Thread-1609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:24,795 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:24,795 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:24,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:24,814 INFO [Thread-1613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1613. 2024-11-09T06:17:24,814 DEBUG [Thread-1613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:24,814 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:24,814 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:24,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 314 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 848 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2501 rows 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 830 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2484 rows 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 831 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2484 rows 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 842 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2499 rows 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 832 2024-11-09T06:17:24,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2486 rows 2024-11-09T06:17:24,815 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T06:17:24,815 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T06:17:24,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:24,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:24,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:24,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T06:17:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T06:17:24,820 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133044820"}]},"ts":"1731133044820"} 2024-11-09T06:17:24,822 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T06:17:24,822 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T06:17:24,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T06:17:24,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, UNASSIGN}] 2024-11-09T06:17:24,825 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, UNASSIGN 2024-11-09T06:17:24,826 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=a2e6477244fddb2a7787090556d9c8bd, regionState=CLOSING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:24,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, UNASSIGN because future has completed 2024-11-09T06:17:24,828 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T06:17:24,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:17:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T06:17:24,981 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:24,981 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T06:17:24,981 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing a2e6477244fddb2a7787090556d9c8bd, disabling compactions & flushes 2024-11-09T06:17:24,981 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:24,981 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:24,981 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. after waiting 0 ms 2024-11-09T06:17:24,981 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:24,981 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing a2e6477244fddb2a7787090556d9c8bd 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T06:17:24,982 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=A 2024-11-09T06:17:24,982 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:24,982 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=B 2024-11-09T06:17:24,982 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:24,982 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2e6477244fddb2a7787090556d9c8bd, store=C 2024-11-09T06:17:24,982 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:24,988 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096920601ee6bc47eb8bedb3f1694d6cfa_a2e6477244fddb2a7787090556d9c8bd is 50, key is test_row_0/A:col10/1731133044793/Put/seqid=0 2024-11-09T06:17:24,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742272_1448 (size=9914) 2024-11-09T06:17:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T06:17:25,395 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:25,406 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096920601ee6bc47eb8bedb3f1694d6cfa_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096920601ee6bc47eb8bedb3f1694d6cfa_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:25,413 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/127faf55bc294ab3b95ad8baaf7beae6, store: [table=TestAcidGuarantees family=A region=a2e6477244fddb2a7787090556d9c8bd] 2024-11-09T06:17:25,413 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/127faf55bc294ab3b95ad8baaf7beae6 is 175, key is test_row_0/A:col10/1731133044793/Put/seqid=0 2024-11-09T06:17:25,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742273_1449 (size=22561) 2024-11-09T06:17:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T06:17:25,820 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=491, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/127faf55bc294ab3b95ad8baaf7beae6 2024-11-09T06:17:25,826 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/1aed955d26c946de952bbcd02e95ad1b is 50, key is test_row_0/B:col10/1731133044793/Put/seqid=0 2024-11-09T06:17:25,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742274_1450 (size=9857) 2024-11-09T06:17:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T06:17:26,229 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/1aed955d26c946de952bbcd02e95ad1b 2024-11-09T06:17:26,236 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/6e77ef42c2764db38461837af904d0dd is 50, key is test_row_0/C:col10/1731133044793/Put/seqid=0 2024-11-09T06:17:26,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742275_1451 (size=9857) 2024-11-09T06:17:26,644 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/6e77ef42c2764db38461837af904d0dd 2024-11-09T06:17:26,648 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/A/127faf55bc294ab3b95ad8baaf7beae6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/127faf55bc294ab3b95ad8baaf7beae6 2024-11-09T06:17:26,653 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/127faf55bc294ab3b95ad8baaf7beae6, entries=100, sequenceid=491, filesize=22.0 K 2024-11-09T06:17:26,654 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/B/1aed955d26c946de952bbcd02e95ad1b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/1aed955d26c946de952bbcd02e95ad1b 2024-11-09T06:17:26,657 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/1aed955d26c946de952bbcd02e95ad1b, entries=100, sequenceid=491, filesize=9.6 K 2024-11-09T06:17:26,658 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/.tmp/C/6e77ef42c2764db38461837af904d0dd as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/6e77ef42c2764db38461837af904d0dd 2024-11-09T06:17:26,661 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/6e77ef42c2764db38461837af904d0dd, entries=100, sequenceid=491, filesize=9.6 K 2024-11-09T06:17:26,662 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a2e6477244fddb2a7787090556d9c8bd in 1681ms, sequenceid=491, compaction requested=true 2024-11-09T06:17:26,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9d50dca1177348a8967b954f296f42dd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/02af962123cf49178ab64275c3013f49, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7ea9902e227244fc9f538edecdd7c0a1, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9966ad1556d3482b846e86584ead59ec, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/a80f317e3b36488d9fe262edf81ef3fe, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7e68d661ea8546a7b611dd8ab1c8a43f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/51871e6623ac43569435bddd62e00e40, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7f4876a08df14e288718948ce8f3ec0b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/bd8bcd5947994ff9b23df1d47f9550b9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/8c64c24af4854507b35c67205b52558c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b730bc71a7204635a0c9b0abcb279fc5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/0158bbc8b166486c9336e87422608a1e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/eb213997cb0745479efa0f41ac36207a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7df7630d58a54121b91021f04166f018, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95ffb0bf5e9b4de1bdd3c8a865cae6d9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/10e4791680ed42b69d1f682a547cbed4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/18b501dae36e4b088aa874e8fb61efc9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/91c123b9a3bb4ee09d77c485a74a7fa4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b307b2f0c38a40a89f0f2bc7589978f6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95b46fc8857d43c7999f4dbb9ae3116b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/dcd72c1ab92140bfb3f7095f80fe7ebf] to archive 2024-11-09T06:17:26,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:17:26,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9d50dca1177348a8967b954f296f42dd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9d50dca1177348a8967b954f296f42dd 2024-11-09T06:17:26,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/02af962123cf49178ab64275c3013f49 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/02af962123cf49178ab64275c3013f49 2024-11-09T06:17:26,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7ea9902e227244fc9f538edecdd7c0a1 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7ea9902e227244fc9f538edecdd7c0a1 2024-11-09T06:17:26,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9966ad1556d3482b846e86584ead59ec to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/9966ad1556d3482b846e86584ead59ec 2024-11-09T06:17:26,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/a80f317e3b36488d9fe262edf81ef3fe to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/a80f317e3b36488d9fe262edf81ef3fe 2024-11-09T06:17:26,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7e68d661ea8546a7b611dd8ab1c8a43f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7e68d661ea8546a7b611dd8ab1c8a43f 2024-11-09T06:17:26,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/51871e6623ac43569435bddd62e00e40 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/51871e6623ac43569435bddd62e00e40 2024-11-09T06:17:26,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7f4876a08df14e288718948ce8f3ec0b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7f4876a08df14e288718948ce8f3ec0b 2024-11-09T06:17:26,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/bd8bcd5947994ff9b23df1d47f9550b9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/bd8bcd5947994ff9b23df1d47f9550b9 2024-11-09T06:17:26,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/8c64c24af4854507b35c67205b52558c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/8c64c24af4854507b35c67205b52558c 2024-11-09T06:17:26,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b730bc71a7204635a0c9b0abcb279fc5 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b730bc71a7204635a0c9b0abcb279fc5 2024-11-09T06:17:26,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/0158bbc8b166486c9336e87422608a1e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/0158bbc8b166486c9336e87422608a1e 2024-11-09T06:17:26,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/eb213997cb0745479efa0f41ac36207a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/eb213997cb0745479efa0f41ac36207a 2024-11-09T06:17:26,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7df7630d58a54121b91021f04166f018 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/7df7630d58a54121b91021f04166f018 2024-11-09T06:17:26,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95ffb0bf5e9b4de1bdd3c8a865cae6d9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95ffb0bf5e9b4de1bdd3c8a865cae6d9 2024-11-09T06:17:26,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/10e4791680ed42b69d1f682a547cbed4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/10e4791680ed42b69d1f682a547cbed4 2024-11-09T06:17:26,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/18b501dae36e4b088aa874e8fb61efc9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/18b501dae36e4b088aa874e8fb61efc9 2024-11-09T06:17:26,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/91c123b9a3bb4ee09d77c485a74a7fa4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/91c123b9a3bb4ee09d77c485a74a7fa4 2024-11-09T06:17:26,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b307b2f0c38a40a89f0f2bc7589978f6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/b307b2f0c38a40a89f0f2bc7589978f6 2024-11-09T06:17:26,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95b46fc8857d43c7999f4dbb9ae3116b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/95b46fc8857d43c7999f4dbb9ae3116b 2024-11-09T06:17:26,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/dcd72c1ab92140bfb3f7095f80fe7ebf to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/dcd72c1ab92140bfb3f7095f80fe7ebf 2024-11-09T06:17:26,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b4589f3b009345bfa00497706a516c75, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/9415b960df3e42d6b6da871f03f0d2d2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/7338403a4d8f4fb5a90e0defd45f0f35, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8e0723249d1e4f8e9c87d2d426d0e0a0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b35dfa42198d424780e5934eb38cd193, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/20abcb8b3e12466dadb218106a7557b8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/0539e6adae4641348b4b71cefc17944f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/08027f7e656548e1a20404a15a5e8a17, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/3ffd67f0505648e599cc9dfd122b13e1, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/ad53b7438a7c419a8a2e8bcb051e1d7f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/5104a52df88d44978eb3b7a74a119f13, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/c2e16a6d5e7549ee82579323b4862d40, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/57031539a4ab4d49afbe978fb333312f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/c2acce878ebe491ebdc0d63b43302440, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/261453f104d44415a6c15b96173e487b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8b59568f498c4f108f6b484287b5337d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8608e84c56384560a41379735b677e75, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/bd9164e992494caa82d769a5788793f3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/03eba2d63e7644bb8a211e974ba88f58, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/69a90ef1c06b40759256c9d9eb682b3d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/e07a03c01ff24cd2a198024a4167cbe6] to archive 2024-11-09T06:17:26,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:17:26,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b4589f3b009345bfa00497706a516c75 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b4589f3b009345bfa00497706a516c75 2024-11-09T06:17:26,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/9415b960df3e42d6b6da871f03f0d2d2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/9415b960df3e42d6b6da871f03f0d2d2 2024-11-09T06:17:26,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/7338403a4d8f4fb5a90e0defd45f0f35 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/7338403a4d8f4fb5a90e0defd45f0f35 2024-11-09T06:17:26,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8e0723249d1e4f8e9c87d2d426d0e0a0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8e0723249d1e4f8e9c87d2d426d0e0a0 2024-11-09T06:17:26,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b35dfa42198d424780e5934eb38cd193 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/b35dfa42198d424780e5934eb38cd193 2024-11-09T06:17:26,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/20abcb8b3e12466dadb218106a7557b8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/20abcb8b3e12466dadb218106a7557b8 2024-11-09T06:17:26,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/0539e6adae4641348b4b71cefc17944f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/0539e6adae4641348b4b71cefc17944f 2024-11-09T06:17:26,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/08027f7e656548e1a20404a15a5e8a17 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/08027f7e656548e1a20404a15a5e8a17 2024-11-09T06:17:26,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/3ffd67f0505648e599cc9dfd122b13e1 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/3ffd67f0505648e599cc9dfd122b13e1 2024-11-09T06:17:26,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/ad53b7438a7c419a8a2e8bcb051e1d7f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/ad53b7438a7c419a8a2e8bcb051e1d7f 2024-11-09T06:17:26,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/5104a52df88d44978eb3b7a74a119f13 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/5104a52df88d44978eb3b7a74a119f13 2024-11-09T06:17:26,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/c2e16a6d5e7549ee82579323b4862d40 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/c2e16a6d5e7549ee82579323b4862d40 2024-11-09T06:17:26,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/57031539a4ab4d49afbe978fb333312f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/57031539a4ab4d49afbe978fb333312f 2024-11-09T06:17:26,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/c2acce878ebe491ebdc0d63b43302440 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/c2acce878ebe491ebdc0d63b43302440 2024-11-09T06:17:26,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/261453f104d44415a6c15b96173e487b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/261453f104d44415a6c15b96173e487b 2024-11-09T06:17:26,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8b59568f498c4f108f6b484287b5337d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8b59568f498c4f108f6b484287b5337d 2024-11-09T06:17:26,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8608e84c56384560a41379735b677e75 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/8608e84c56384560a41379735b677e75 2024-11-09T06:17:26,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/bd9164e992494caa82d769a5788793f3 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/bd9164e992494caa82d769a5788793f3 2024-11-09T06:17:26,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/03eba2d63e7644bb8a211e974ba88f58 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/03eba2d63e7644bb8a211e974ba88f58 2024-11-09T06:17:26,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/69a90ef1c06b40759256c9d9eb682b3d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/69a90ef1c06b40759256c9d9eb682b3d 2024-11-09T06:17:26,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/e07a03c01ff24cd2a198024a4167cbe6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/e07a03c01ff24cd2a198024a4167cbe6 2024-11-09T06:17:26,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/02bccfae5115403f86a6f83eb8e1df84, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/66c2616ad0ee4904a88915d220d568a5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/9d2058c8f8364c34b7b4b255cc034a89, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/85ca7d9aa5994d4387fadd12f30f8015, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/2eb0b4931a9745cc97dc9c8793276b35, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/86dce45162f34844ac54dee1258b0525, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/e4cfb9cb892d4380b569206a88a1ca52, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/ef01a8ddd40b43548dec1b38fec797d0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/f4bf968b731f49c6952cd4a151cf28f2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/0f9d01e01582499ab467ded2ce73714b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/b5a1b81dc96a47e28f189fb3bd5e0ffd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/08ae04031e09406a900effd26db50dcb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/a52e253b27a7492dadacaa4ab1e60724, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/dd93bfac476c413fab5fc982004e40d4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/135188a920fb4fa7b567e8b0efbad861, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/8dd4be4beb074b478389fdc68633e0e1, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/12e5db32869e428ab94fe02606c256ed, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/f2159634dcbc49e1b8ca8043bdda1084, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/bb033ea6f9bc43aca54924e831fd102a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/60bed4e31b374cad90c1842d67922bbe, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/5c7485ddd323468ca78c2f8a93af5c0c] to archive 2024-11-09T06:17:26,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:17:26,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/02bccfae5115403f86a6f83eb8e1df84 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/02bccfae5115403f86a6f83eb8e1df84 2024-11-09T06:17:26,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/66c2616ad0ee4904a88915d220d568a5 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/66c2616ad0ee4904a88915d220d568a5 2024-11-09T06:17:26,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/9d2058c8f8364c34b7b4b255cc034a89 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/9d2058c8f8364c34b7b4b255cc034a89 2024-11-09T06:17:26,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/85ca7d9aa5994d4387fadd12f30f8015 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/85ca7d9aa5994d4387fadd12f30f8015 2024-11-09T06:17:26,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/2eb0b4931a9745cc97dc9c8793276b35 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/2eb0b4931a9745cc97dc9c8793276b35 2024-11-09T06:17:26,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/86dce45162f34844ac54dee1258b0525 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/86dce45162f34844ac54dee1258b0525 2024-11-09T06:17:26,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/e4cfb9cb892d4380b569206a88a1ca52 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/e4cfb9cb892d4380b569206a88a1ca52 2024-11-09T06:17:26,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/ef01a8ddd40b43548dec1b38fec797d0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/ef01a8ddd40b43548dec1b38fec797d0 2024-11-09T06:17:26,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/f4bf968b731f49c6952cd4a151cf28f2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/f4bf968b731f49c6952cd4a151cf28f2 2024-11-09T06:17:26,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/0f9d01e01582499ab467ded2ce73714b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/0f9d01e01582499ab467ded2ce73714b 2024-11-09T06:17:26,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/b5a1b81dc96a47e28f189fb3bd5e0ffd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/b5a1b81dc96a47e28f189fb3bd5e0ffd 2024-11-09T06:17:26,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/08ae04031e09406a900effd26db50dcb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/08ae04031e09406a900effd26db50dcb 2024-11-09T06:17:26,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/a52e253b27a7492dadacaa4ab1e60724 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/a52e253b27a7492dadacaa4ab1e60724 2024-11-09T06:17:26,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/dd93bfac476c413fab5fc982004e40d4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/dd93bfac476c413fab5fc982004e40d4 2024-11-09T06:17:26,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/135188a920fb4fa7b567e8b0efbad861 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/135188a920fb4fa7b567e8b0efbad861 2024-11-09T06:17:26,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/8dd4be4beb074b478389fdc68633e0e1 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/8dd4be4beb074b478389fdc68633e0e1 2024-11-09T06:17:26,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/12e5db32869e428ab94fe02606c256ed to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/12e5db32869e428ab94fe02606c256ed 2024-11-09T06:17:26,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/f2159634dcbc49e1b8ca8043bdda1084 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/f2159634dcbc49e1b8ca8043bdda1084 2024-11-09T06:17:26,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/bb033ea6f9bc43aca54924e831fd102a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/bb033ea6f9bc43aca54924e831fd102a 2024-11-09T06:17:26,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/60bed4e31b374cad90c1842d67922bbe to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/60bed4e31b374cad90c1842d67922bbe 2024-11-09T06:17:26,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/5c7485ddd323468ca78c2f8a93af5c0c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/5c7485ddd323468ca78c2f8a93af5c0c 2024-11-09T06:17:26,784 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/recovered.edits/494.seqid, newMaxSeqId=494, maxSeqId=4 2024-11-09T06:17:26,784 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. 2024-11-09T06:17:26,784 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for a2e6477244fddb2a7787090556d9c8bd: Waiting for close lock at 1731133044981Running coprocessor pre-close hooks at 1731133044981Disabling compacts and flushes for region at 1731133044981Disabling writes for close at 1731133044981Obtaining lock to block concurrent updates at 1731133044981Preparing flush snapshotting stores in a2e6477244fddb2a7787090556d9c8bd at 1731133044981Finished memstore snapshotting TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731133044982 (+1 ms)Flushing stores of TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd. at 1731133044982Flushing a2e6477244fddb2a7787090556d9c8bd/A: creating writer at 1731133044982Flushing mob file a2e6477244fddb2a7787090556d9c8bd/A: appending metadata at 1731133044988 (+6 ms)Flushing mob file a2e6477244fddb2a7787090556d9c8bd/A: closing flushed file at 1731133044988Flushing a2e6477244fddb2a7787090556d9c8bd/A: appending metadata at 1731133045413 (+425 ms)Flushing a2e6477244fddb2a7787090556d9c8bd/A: closing flushed file at 1731133045413Flushing a2e6477244fddb2a7787090556d9c8bd/B: creating writer at 1731133045823 (+410 ms)Flushing a2e6477244fddb2a7787090556d9c8bd/B: appending metadata at 1731133045825 (+2 ms)Flushing a2e6477244fddb2a7787090556d9c8bd/B: closing flushed file at 1731133045825Flushing a2e6477244fddb2a7787090556d9c8bd/C: creating writer at 1731133046233 (+408 ms)Flushing a2e6477244fddb2a7787090556d9c8bd/C: appending metadata at 1731133046235 (+2 ms)Flushing a2e6477244fddb2a7787090556d9c8bd/C: closing flushed file at 1731133046235Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67e5ec25: reopening flushed file at 1731133046647 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f0aaa00: reopening flushed file at 1731133046653 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d576dda: reopening flushed file at 1731133046657 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a2e6477244fddb2a7787090556d9c8bd in 1681ms, sequenceid=491, compaction requested=true at 1731133046662 (+5 ms)Writing region close event to WAL at 1731133046781 (+119 ms)Running coprocessor post-close hooks at 1731133046784 (+3 ms)Closed at 1731133046784 2024-11-09T06:17:26,787 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:26,787 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=a2e6477244fddb2a7787090556d9c8bd, regionState=CLOSED 2024-11-09T06:17:26,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:17:26,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-09T06:17:26,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure a2e6477244fddb2a7787090556d9c8bd, server=4a84bcce7bad,46317,1731132929173 in 1.9620 sec 2024-11-09T06:17:26,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-09T06:17:26,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2e6477244fddb2a7787090556d9c8bd, UNASSIGN in 1.9680 sec 2024-11-09T06:17:26,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-09T06:17:26,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9720 sec 2024-11-09T06:17:26,800 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133046800"}]},"ts":"1731133046800"} 2024-11-09T06:17:26,802 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T06:17:26,802 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T06:17:26,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9860 sec 2024-11-09T06:17:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T06:17:26,963 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:26,964 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T06:17:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:26,966 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T06:17:26,967 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:26,976 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:26,979 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/recovered.edits] 2024-11-09T06:17:26,981 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/10382159871a4cda8155d00d608f02c5 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/10382159871a4cda8155d00d608f02c5 2024-11-09T06:17:26,982 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/127faf55bc294ab3b95ad8baaf7beae6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/127faf55bc294ab3b95ad8baaf7beae6 2024-11-09T06:17:26,983 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/3ae012b322af4cd79f0e8f4d32881658 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/A/3ae012b322af4cd79f0e8f4d32881658 2024-11-09T06:17:26,985 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/1aed955d26c946de952bbcd02e95ad1b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/1aed955d26c946de952bbcd02e95ad1b 2024-11-09T06:17:26,985 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/7c048c5b1e404bcdb505f4988edee682 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/7c048c5b1e404bcdb505f4988edee682 2024-11-09T06:17:26,986 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/fe52308e439f43838de78061707a7e5b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/B/fe52308e439f43838de78061707a7e5b 2024-11-09T06:17:26,987 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/56a39cc60bdb4645a1e11d52c0115844 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/56a39cc60bdb4645a1e11d52c0115844 2024-11-09T06:17:26,988 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/6e77ef42c2764db38461837af904d0dd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/6e77ef42c2764db38461837af904d0dd 2024-11-09T06:17:26,990 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/9ee4de50fb9b427997983a3e7842965d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/C/9ee4de50fb9b427997983a3e7842965d 2024-11-09T06:17:26,993 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/recovered.edits/494.seqid to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd/recovered.edits/494.seqid 2024-11-09T06:17:26,994 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:26,994 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T06:17:26,995 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T06:17:26,995 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T06:17:26,999 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109007f1e4d4f164014b550493ec52d8cda_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109007f1e4d4f164014b550493ec52d8cda_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,001 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110926b985f4b4124d9f8d73f62baeaa724c_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110926b985f4b4124d9f8d73f62baeaa724c_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,002 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110936f4b782fd6b40a3aa29880de5bc353a_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110936f4b782fd6b40a3aa29880de5bc353a_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,003 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093a229450f0dc4faf98ad6da74f96f81b_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093a229450f0dc4faf98ad6da74f96f81b_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,005 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093bf9e130ed6047caace734b227856b7c_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093bf9e130ed6047caace734b227856b7c_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,007 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110943c06bb04bb645c2b533134d43331a20_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110943c06bb04bb645c2b533134d43331a20_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,010 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110944f508b97af0473f8db390f0c4cf38d0_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110944f508b97af0473f8db390f0c4cf38d0_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,012 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109464d41c2b84b4d7e8aef5bdbdc80ba1f_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109464d41c2b84b4d7e8aef5bdbdc80ba1f_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,013 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095824b6c374d240d888baeb40ed42c28d_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095824b6c374d240d888baeb40ed42c28d_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,014 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096920601ee6bc47eb8bedb3f1694d6cfa_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096920601ee6bc47eb8bedb3f1694d6cfa_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,015 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109731558e735d74a6f9039164425cb631a_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109731558e735d74a6f9039164425cb631a_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,021 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097f446c0f2c2f43d9beea81cb068a4bae_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097f446c0f2c2f43d9beea81cb068a4bae_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,022 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110991094308e9514459a72d9a4c17a95898_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110991094308e9514459a72d9a4c17a95898_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,023 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a3af045ce41541dc925859f27c9445c7_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a3af045ce41541dc925859f27c9445c7_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,024 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e0c04dfa2b3d4a068a47d11cc0cecb86_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e0c04dfa2b3d4a068a47d11cc0cecb86_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,027 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e6470981a53b489fb86b4e9c64d77208_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e6470981a53b489fb86b4e9c64d77208_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,028 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eb090b3f0ab0432d8fab94d8ae8cccd1_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eb090b3f0ab0432d8fab94d8ae8cccd1_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,030 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f0cb17415db940d58594517643673664_a2e6477244fddb2a7787090556d9c8bd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f0cb17415db940d58594517643673664_a2e6477244fddb2a7787090556d9c8bd 2024-11-09T06:17:27,031 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T06:17:27,035 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:27,037 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T06:17:27,039 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T06:17:27,043 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:27,043 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T06:17:27,043 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731133047043"}]},"ts":"9223372036854775807"} 2024-11-09T06:17:27,045 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T06:17:27,045 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a2e6477244fddb2a7787090556d9c8bd, NAME => 'TestAcidGuarantees,,1731133019174.a2e6477244fddb2a7787090556d9c8bd.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T06:17:27,045 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T06:17:27,046 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731133047045"}]},"ts":"9223372036854775807"} 2024-11-09T06:17:27,048 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T06:17:27,049 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:27,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-11-09T06:17:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T06:17:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T06:17:27,078 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:27,092 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 249) - Thread LEAK? -, OpenFileDescriptor=463 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 805) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=945 (was 1519) 2024-11-09T06:17:27,101 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=944 2024-11-09T06:17:27,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T06:17:27,102 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T06:17:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:27,104 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T06:17:27,104 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:27,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-11-09T06:17:27,105 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T06:17:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-09T06:17:27,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742276_1452 (size=1061) 2024-11-09T06:17:27,118 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f 2024-11-09T06:17:27,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742277_1453 (size=53) 2024-11-09T06:17:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-09T06:17:27,299 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T06:17:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-09T06:17:27,527 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:17:27,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f15639486d79cf453f8c760f89863e07, disabling compactions & flushes 2024-11-09T06:17:27,528 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:27,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:27,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. after waiting 0 ms 2024-11-09T06:17:27,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:27,528 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:27,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f15639486d79cf453f8c760f89863e07: Waiting for close lock at 1731133047528Disabling compacts and flushes for region at 1731133047528Disabling writes for close at 1731133047528Writing region close event to WAL at 1731133047528Closed at 1731133047528 2024-11-09T06:17:27,529 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T06:17:27,529 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731133047529"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731133047529"}]},"ts":"1731133047529"} 2024-11-09T06:17:27,530 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T06:17:27,531 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T06:17:27,531 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133047531"}]},"ts":"1731133047531"} 2024-11-09T06:17:27,532 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T06:17:27,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f15639486d79cf453f8c760f89863e07, ASSIGN}] 2024-11-09T06:17:27,534 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f15639486d79cf453f8c760f89863e07, ASSIGN 2024-11-09T06:17:27,534 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f15639486d79cf453f8c760f89863e07, ASSIGN; state=OFFLINE, location=4a84bcce7bad,46317,1731132929173; forceNewPlan=false, retain=false 2024-11-09T06:17:27,685 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=f15639486d79cf453f8c760f89863e07, regionState=OPENING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:27,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f15639486d79cf453f8c760f89863e07, ASSIGN because future has completed 2024-11-09T06:17:27,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:17:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-09T06:17:27,841 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:27,841 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} 2024-11-09T06:17:27,842 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,842 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:17:27,842 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,842 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,844 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,845 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:27,845 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f15639486d79cf453f8c760f89863e07 columnFamilyName A 2024-11-09T06:17:27,845 DEBUG [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:27,846 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] regionserver.HStore(327): Store=f15639486d79cf453f8c760f89863e07/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:27,847 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,850 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:27,850 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f15639486d79cf453f8c760f89863e07 columnFamilyName B 2024-11-09T06:17:27,850 DEBUG [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:27,851 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] regionserver.HStore(327): Store=f15639486d79cf453f8c760f89863e07/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:27,851 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,852 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:27,852 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f15639486d79cf453f8c760f89863e07 columnFamilyName C 2024-11-09T06:17:27,852 DEBUG [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:27,853 INFO [StoreOpener-f15639486d79cf453f8c760f89863e07-1 {}] regionserver.HStore(327): Store=f15639486d79cf453f8c760f89863e07/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:27,853 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:27,853 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,854 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,854 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,855 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,855 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,856 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T06:17:27,857 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,858 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T06:17:27,859 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened f15639486d79cf453f8c760f89863e07; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75090324, jitterRate=0.11893302202224731}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T06:17:27,859 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:27,859 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for f15639486d79cf453f8c760f89863e07: Running coprocessor pre-open hook at 1731133047842Writing region info on filesystem at 1731133047842Initializing all the Stores at 1731133047843 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133047843Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133047843Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133047843Cleaning up temporary data from old regions at 1731133047855 (+12 ms)Running coprocessor post-open hooks at 1731133047859 (+4 ms)Region opened successfully at 1731133047859 2024-11-09T06:17:27,860 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., pid=110, masterSystemTime=1731133047838 2024-11-09T06:17:27,862 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:27,862 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:27,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=f15639486d79cf453f8c760f89863e07, regionState=OPEN, openSeqNum=2, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:27,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:17:27,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-09T06:17:27,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 in 177 msec 2024-11-09T06:17:27,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-09T06:17:27,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f15639486d79cf453f8c760f89863e07, ASSIGN in 333 msec 2024-11-09T06:17:27,868 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T06:17:27,868 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133047868"}]},"ts":"1731133047868"} 2024-11-09T06:17:27,869 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T06:17:27,870 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T06:17:27,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 768 msec 2024-11-09T06:17:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-09T06:17:28,243 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:28,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:28,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:28,244 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11694ee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:28,246 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39f961a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,246 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:28,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:28,248 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T06:17:28,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a84bcce7bad,37069,1731132928314 2024-11-09T06:17:28,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e94c014 2024-11-09T06:17:28,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T06:17:28,250 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T06:17:28,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:28,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:28,251 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121a9b57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:28,252 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15233e81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:28,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:28,254 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42603af7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:28,255 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d09afe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:28,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:28,256 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f456e70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:28,257 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18099f39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:28,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:28,259 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:28,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:28,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:28,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb3ceae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:28,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:28,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:28,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:28,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@589c1dc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:28,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:28,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:28,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:28,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:28,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23762fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:28,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:28,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:28,262 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@697d16b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:28,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:28,263 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6540f37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:28,264 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d80eb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:28,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:28,266 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3ff71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:28,267 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f3dacca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:28,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:28,269 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283a75dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:28,270 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297604bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,271 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:28,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:28,271 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1223f7c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:28,273 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1023c341, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:28,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:28,274 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b122ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:28,275 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9874ed0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:28,276 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:28,277 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:28,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:28,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:17:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:17:28,283 DEBUG [Thread-2047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:28,283 DEBUG [Thread-2039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:28,283 DEBUG [Thread-2028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:28,284 DEBUG [Thread-2045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:28,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:28,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:28,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:17:28,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:17:28,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:28,289 DEBUG [Thread-2036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:28,290 DEBUG [Thread-2034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:28,290 DEBUG [Thread-2041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:17:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:17:28,291 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-09T06:17:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:28,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:17:28,293 DEBUG [Thread-2043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:28,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:28,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:28,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:28,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:28,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:28,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:28,294 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:28,295 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:28,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:28,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:28,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:28,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T06:17:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:28,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:17:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:17:28,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:17:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2] 2024-11-09T06:17:28,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/8fb90c5f28084a4f804bc4440eda6a16 is 50, key is test_row_0/A:col10/1731133048284/Put/seqid=0 2024-11-09T06:17:28,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742278_1454 (size=12001) 2024-11-09T06:17:28,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133048339, max timestamp=1731133048357Num uniques 859; 2024-11-09T06:17:28,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:28,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133048339, max timestamp=1731133048357Num uniques 859; 2024-11-09T06:17:28,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:28,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133048339, max timestamp=1731133048357Num uniques 859; 2024-11-09T06:17:28,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:28,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133108389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T06:17:28,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133108390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133108391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133108393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133108391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T06:17:28,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:28,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:28,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:28,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133108503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133108513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133108513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133108513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133108513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T06:17:28,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:28,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:28,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:28,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T06:17:28,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133108712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133108733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133108733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133108733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133108733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:28,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/8fb90c5f28084a4f804bc4440eda6a16 2024-11-09T06:17:28,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T06:17:28,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:28,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:28,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/79657644920449ef846298c575d28c3f is 50, key is test_row_0/B:col10/1731133048284/Put/seqid=0 2024-11-09T06:17:28,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742279_1455 (size=12001) 2024-11-09T06:17:28,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T06:17:28,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:28,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:28,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:28,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T06:17:29,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133109034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:29,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:29,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133109042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:29,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:29,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133109043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:29,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133109043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:29,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133109053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:29,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T06:17:29,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:29,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:29,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:29,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/79657644920449ef846298c575d28c3f 2024-11-09T06:17:29,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/f94cf85188494705b1d8baa435aae6dd is 50, key is test_row_0/C:col10/1731133048284/Put/seqid=0 2024-11-09T06:17:29,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T06:17:29,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:29,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:29,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:29,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742280_1456 (size=12001) 2024-11-09T06:17:29,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T06:17:29,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:29,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:29,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:29,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T06:17:29,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T06:17:29,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:29,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133109553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:29,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133109563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:29,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133109563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133109563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133109564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:29,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/f94cf85188494705b1d8baa435aae6dd 2024-11-09T06:17:29,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/8fb90c5f28084a4f804bc4440eda6a16 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8fb90c5f28084a4f804bc4440eda6a16 2024-11-09T06:17:29,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8fb90c5f28084a4f804bc4440eda6a16, entries=150, sequenceid=11, filesize=11.7 K 2024-11-09T06:17:29,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/79657644920449ef846298c575d28c3f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/79657644920449ef846298c575d28c3f 2024-11-09T06:17:29,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/79657644920449ef846298c575d28c3f, entries=150, sequenceid=11, filesize=11.7 K 2024-11-09T06:17:29,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/f94cf85188494705b1d8baa435aae6dd as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/f94cf85188494705b1d8baa435aae6dd 2024-11-09T06:17:29,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T06:17:29,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:29,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:29,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:29,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/f94cf85188494705b1d8baa435aae6dd, entries=150, sequenceid=11, filesize=11.7 K 2024-11-09T06:17:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:29,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f15639486d79cf453f8c760f89863e07 in 1385ms, sequenceid=11, compaction requested=false 2024-11-09T06:17:29,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:29,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T06:17:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:29,830 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-09T06:17:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/7a63113aafe1440bbc86b677c7387137 is 50, key is test_row_0/A:col10/1731133048390/Put/seqid=0 2024-11-09T06:17:29,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742281_1457 (size=12001) 2024-11-09T06:17:29,925 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/7a63113aafe1440bbc86b677c7387137 2024-11-09T06:17:29,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/375a2a9c82b54a328559b6abbdc164c6 is 50, key is test_row_0/B:col10/1731133048390/Put/seqid=0 2024-11-09T06:17:29,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742282_1458 (size=12001) 2024-11-09T06:17:29,999 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/375a2a9c82b54a328559b6abbdc164c6 2024-11-09T06:17:30,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/726b29947d4c4308bb78d6ca84fe24da is 50, key is test_row_0/C:col10/1731133048390/Put/seqid=0 2024-11-09T06:17:30,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742283_1459 (size=12001) 2024-11-09T06:17:30,088 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/726b29947d4c4308bb78d6ca84fe24da 2024-11-09T06:17:30,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/7a63113aafe1440bbc86b677c7387137 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/7a63113aafe1440bbc86b677c7387137 2024-11-09T06:17:30,102 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/7a63113aafe1440bbc86b677c7387137, entries=150, sequenceid=47, filesize=11.7 K 2024-11-09T06:17:30,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/375a2a9c82b54a328559b6abbdc164c6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/375a2a9c82b54a328559b6abbdc164c6 2024-11-09T06:17:30,114 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/375a2a9c82b54a328559b6abbdc164c6, entries=150, sequenceid=47, filesize=11.7 K 2024-11-09T06:17:30,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/726b29947d4c4308bb78d6ca84fe24da as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/726b29947d4c4308bb78d6ca84fe24da 2024-11-09T06:17:30,122 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/726b29947d4c4308bb78d6ca84fe24da, entries=150, sequenceid=47, filesize=11.7 K 2024-11-09T06:17:30,124 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for f15639486d79cf453f8c760f89863e07 in 294ms, sequenceid=47, compaction requested=false 2024-11-09T06:17:30,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:30,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:30,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-09T06:17:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-09T06:17:30,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-09T06:17:30,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-11-09T06:17:30,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8410 sec 2024-11-09T06:17:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T06:17:30,455 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:30,456 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-09T06:17:30,458 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T06:17:30,459 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:30,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T06:17:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:30,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:17:30,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:30,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:30,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:30,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:30,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:30,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:30,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-09T06:17:30,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:30,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:30,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:30,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:30,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:30,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/5bbe22f6615349a69ad22769160e67e2 is 50, key is test_row_0/A:col10/1731133050596/Put/seqid=0 2024-11-09T06:17:30,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133050636, max timestamp=1731133050655Num uniques 859; 2024-11-09T06:17:30,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:30,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133110648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:30,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133050636, max timestamp=1731133050655Num uniques 859; 2024-11-09T06:17:30,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133050636, max timestamp=1731133050655Num uniques 859; 2024-11-09T06:17:30,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:30,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:30,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742284_1460 (size=16681) 2024-11-09T06:17:30,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/5bbe22f6615349a69ad22769160e67e2 2024-11-09T06:17:30,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/ef2a6b4932864eb9974b39edba5fb24d is 50, key is test_row_0/B:col10/1731133050596/Put/seqid=0 2024-11-09T06:17:30,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742285_1461 (size=12001) 2024-11-09T06:17:30,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133110743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133110743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133110759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133110759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:30,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-09T06:17:30,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:30,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:30,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:30,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:30,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T06:17:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133110764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:30,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133110873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133110873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:30,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133110873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133110873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:30,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-09T06:17:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:30,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:30,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133110982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-09T06:17:31,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:31,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T06:17:31,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/ef2a6b4932864eb9974b39edba5fb24d 2024-11-09T06:17:31,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/54d480088cd44f1d8e23731ae9efcb31 is 50, key is test_row_0/C:col10/1731133050596/Put/seqid=0 2024-11-09T06:17:31,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133111093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133111093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133111093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133111093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742286_1462 (size=12001) 2024-11-09T06:17:31,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-09T06:17:31,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:31,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133111303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-09T06:17:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:31,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133111413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133111413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133111413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133111422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/54d480088cd44f1d8e23731ae9efcb31 2024-11-09T06:17:31,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/5bbe22f6615349a69ad22769160e67e2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5bbe22f6615349a69ad22769160e67e2 2024-11-09T06:17:31,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5bbe22f6615349a69ad22769160e67e2, entries=250, sequenceid=57, filesize=16.3 K 2024-11-09T06:17:31,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/ef2a6b4932864eb9974b39edba5fb24d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ef2a6b4932864eb9974b39edba5fb24d 2024-11-09T06:17:31,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ef2a6b4932864eb9974b39edba5fb24d, entries=150, sequenceid=57, filesize=11.7 K 2024-11-09T06:17:31,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/54d480088cd44f1d8e23731ae9efcb31 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/54d480088cd44f1d8e23731ae9efcb31 2024-11-09T06:17:31,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-09T06:17:31,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:31,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:31,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/54d480088cd44f1d8e23731ae9efcb31, entries=150, sequenceid=57, filesize=11.7 K 2024-11-09T06:17:31,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f15639486d79cf453f8c760f89863e07 in 949ms, sequenceid=57, compaction requested=true 2024-11-09T06:17:31,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:31,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:31,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:31,549 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:31,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:31,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:31,550 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:31,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:31,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:31,550 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:31,550 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:31,550 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/B is initiating minor compaction (all files) 2024-11-09T06:17:31,551 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/A is initiating minor compaction (all files) 2024-11-09T06:17:31,550 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/B in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,551 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/A in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,551 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/79657644920449ef846298c575d28c3f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/375a2a9c82b54a328559b6abbdc164c6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ef2a6b4932864eb9974b39edba5fb24d] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=35.2 K 2024-11-09T06:17:31,551 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8fb90c5f28084a4f804bc4440eda6a16, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/7a63113aafe1440bbc86b677c7387137, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5bbe22f6615349a69ad22769160e67e2] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=39.7 K 2024-11-09T06:17:31,551 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 79657644920449ef846298c575d28c3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731133048284 2024-11-09T06:17:31,551 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fb90c5f28084a4f804bc4440eda6a16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731133048284 2024-11-09T06:17:31,552 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 375a2a9c82b54a328559b6abbdc164c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731133048378 2024-11-09T06:17:31,552 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a63113aafe1440bbc86b677c7387137, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731133048378 2024-11-09T06:17:31,552 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting ef2a6b4932864eb9974b39edba5fb24d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731133050586 2024-11-09T06:17:31,552 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bbe22f6615349a69ad22769160e67e2, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731133050585 2024-11-09T06:17:31,574 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#A#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:31,575 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/044e2222055c4703a2a216c7e6a076ec is 50, key is test_row_0/A:col10/1731133050596/Put/seqid=0 2024-11-09T06:17:31,584 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#B#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:31,584 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/2a3f4b7de6364c5594a63a73ca4d5874 is 50, key is test_row_0/B:col10/1731133050596/Put/seqid=0 2024-11-09T06:17:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T06:17:31,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742287_1463 (size=12104) 2024-11-09T06:17:31,615 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/044e2222055c4703a2a216c7e6a076ec as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/044e2222055c4703a2a216c7e6a076ec 2024-11-09T06:17:31,619 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/A of f15639486d79cf453f8c760f89863e07 into 044e2222055c4703a2a216c7e6a076ec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:31,619 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:31,619 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/A, priority=13, startTime=1731133051549; duration=0sec 2024-11-09T06:17:31,619 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:31,619 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:A 2024-11-09T06:17:31,619 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:31,621 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:31,621 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/C is initiating minor compaction (all files) 2024-11-09T06:17:31,621 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/C in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,622 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/f94cf85188494705b1d8baa435aae6dd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/726b29947d4c4308bb78d6ca84fe24da, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/54d480088cd44f1d8e23731ae9efcb31] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=35.2 K 2024-11-09T06:17:31,622 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f94cf85188494705b1d8baa435aae6dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731133048284 2024-11-09T06:17:31,623 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 726b29947d4c4308bb78d6ca84fe24da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731133048378 2024-11-09T06:17:31,623 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54d480088cd44f1d8e23731ae9efcb31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731133050586 2024-11-09T06:17:31,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742288_1464 (size=12104) 2024-11-09T06:17:31,646 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#C#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:31,647 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/792ac5c4a00448fca2c0bd35c16b3c16 is 50, key is test_row_0/C:col10/1731133050596/Put/seqid=0 2024-11-09T06:17:31,659 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/2a3f4b7de6364c5594a63a73ca4d5874 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/2a3f4b7de6364c5594a63a73ca4d5874 2024-11-09T06:17:31,671 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/B of f15639486d79cf453f8c760f89863e07 into 2a3f4b7de6364c5594a63a73ca4d5874(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:31,671 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:31,671 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/B, priority=13, startTime=1731133051549; duration=0sec 2024-11-09T06:17:31,671 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:31,671 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:B 2024-11-09T06:17:31,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742289_1465 (size=12104) 2024-11-09T06:17:31,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-09T06:17:31,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:31,697 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-09T06:17:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:31,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/555ee8121df2464cac3437c89ffe5e5e is 50, key is test_row_0/A:col10/1731133050757/Put/seqid=0 2024-11-09T06:17:31,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742290_1466 (size=12001) 2024-11-09T06:17:31,729 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/555ee8121df2464cac3437c89ffe5e5e 2024-11-09T06:17:31,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/05b6c7ef536044f39cfeccdf5c4015a2 is 50, key is test_row_0/B:col10/1731133050757/Put/seqid=0 2024-11-09T06:17:31,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742291_1467 (size=12001) 2024-11-09T06:17:31,810 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/05b6c7ef536044f39cfeccdf5c4015a2 2024-11-09T06:17:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:31,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:31,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/ab6fc06aeef44b91a76165c850ae11d4 is 50, key is test_row_0/C:col10/1731133050757/Put/seqid=0 2024-11-09T06:17:31,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742292_1468 (size=12001) 2024-11-09T06:17:31,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133111883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133111932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133111932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133111932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:31,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133111943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:32,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133112002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:32,090 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/792ac5c4a00448fca2c0bd35c16b3c16 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/792ac5c4a00448fca2c0bd35c16b3c16 2024-11-09T06:17:32,094 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/C of f15639486d79cf453f8c760f89863e07 into 792ac5c4a00448fca2c0bd35c16b3c16(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:32,094 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:32,094 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/C, priority=13, startTime=1731133051550; duration=0sec 2024-11-09T06:17:32,094 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:32,094 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:C 2024-11-09T06:17:32,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133112222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:32,253 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/ab6fc06aeef44b91a76165c850ae11d4 2024-11-09T06:17:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/555ee8121df2464cac3437c89ffe5e5e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/555ee8121df2464cac3437c89ffe5e5e 2024-11-09T06:17:32,260 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/555ee8121df2464cac3437c89ffe5e5e, entries=150, sequenceid=96, filesize=11.7 K 2024-11-09T06:17:32,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/05b6c7ef536044f39cfeccdf5c4015a2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/05b6c7ef536044f39cfeccdf5c4015a2 2024-11-09T06:17:32,265 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/05b6c7ef536044f39cfeccdf5c4015a2, entries=150, sequenceid=96, filesize=11.7 K 2024-11-09T06:17:32,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/ab6fc06aeef44b91a76165c850ae11d4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/ab6fc06aeef44b91a76165c850ae11d4 2024-11-09T06:17:32,269 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/ab6fc06aeef44b91a76165c850ae11d4, entries=150, sequenceid=96, filesize=11.7 K 2024-11-09T06:17:32,270 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for f15639486d79cf453f8c760f89863e07 in 572ms, sequenceid=96, compaction requested=false 2024-11-09T06:17:32,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:32,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:32,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-09T06:17:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-09T06:17:32,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-09T06:17:32,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8120 sec 2024-11-09T06:17:32,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.8170 sec 2024-11-09T06:17:32,332 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T06:17:32,497 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8fb90c5f28084a4f804bc4440eda6a16, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/7a63113aafe1440bbc86b677c7387137, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5bbe22f6615349a69ad22769160e67e2] to archive 2024-11-09T06:17:32,498 WARN [regionserver/4a84bcce7bad:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-09T06:17:32,507 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:17:32,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8fb90c5f28084a4f804bc4440eda6a16 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8fb90c5f28084a4f804bc4440eda6a16 2024-11-09T06:17:32,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/7a63113aafe1440bbc86b677c7387137 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/7a63113aafe1440bbc86b677c7387137 2024-11-09T06:17:32,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5bbe22f6615349a69ad22769160e67e2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5bbe22f6615349a69ad22769160e67e2 2024-11-09T06:17:32,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/79657644920449ef846298c575d28c3f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/375a2a9c82b54a328559b6abbdc164c6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ef2a6b4932864eb9974b39edba5fb24d] to archive 2024-11-09T06:17:32,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:17:32,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/79657644920449ef846298c575d28c3f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/79657644920449ef846298c575d28c3f 2024-11-09T06:17:32,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/375a2a9c82b54a328559b6abbdc164c6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/375a2a9c82b54a328559b6abbdc164c6 2024-11-09T06:17:32,524 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ef2a6b4932864eb9974b39edba5fb24d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ef2a6b4932864eb9974b39edba5fb24d 2024-11-09T06:17:32,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/f94cf85188494705b1d8baa435aae6dd, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/726b29947d4c4308bb78d6ca84fe24da, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/54d480088cd44f1d8e23731ae9efcb31] to archive 2024-11-09T06:17:32,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:17:32,534 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/f94cf85188494705b1d8baa435aae6dd to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/f94cf85188494705b1d8baa435aae6dd 2024-11-09T06:17:32,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/726b29947d4c4308bb78d6ca84fe24da to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/726b29947d4c4308bb78d6ca84fe24da 2024-11-09T06:17:32,538 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/4a84bcce7bad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/54d480088cd44f1d8e23731ae9efcb31 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/54d480088cd44f1d8e23731ae9efcb31 2024-11-09T06:17:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:32,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:17:32,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:32,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:32,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:32,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:32,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:32,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:32,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/2be67310c49f499ea2ab7f4a2eb5ea84 is 50, key is test_row_0/A:col10/1731133051881/Put/seqid=0 2024-11-09T06:17:32,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742293_1469 (size=12001) 2024-11-09T06:17:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T06:17:32,603 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:32,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-09T06:17:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T06:17:32,607 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:32,608 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:32,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T06:17:32,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T06:17:32,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:32,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:32,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:32,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:32,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:32,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133052654, max timestamp=1731133052770Num uniques 859; 2024-11-09T06:17:32,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133052654, max timestamp=1731133052770Num uniques 859; 2024-11-09T06:17:32,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:32,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:32,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133052654, max timestamp=1731133052770Num uniques 859; 2024-11-09T06:17:32,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:32,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T06:17:32,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:32,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:32,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:32,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T06:17:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133112934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:32,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133112942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:32,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133112942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:32,952 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in f15639486d79cf453f8c760f89863e07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:32,952 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in f15639486d79cf453f8c760f89863e07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:32,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:32,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:32,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133112953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:32,966 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in f15639486d79cf453f8c760f89863e07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:32,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133112962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:32,977 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in f15639486d79cf453f8c760f89863e07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:32,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/2be67310c49f499ea2ab7f4a2eb5ea84 2024-11-09T06:17:33,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/28a60559a0f54d2c9b24be225c3efe8d is 50, key is test_row_0/B:col10/1731133051881/Put/seqid=0 2024-11-09T06:17:33,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742294_1470 (size=12001) 2024-11-09T06:17:33,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/28a60559a0f54d2c9b24be225c3efe8d 2024-11-09T06:17:33,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/7d8651ef67b240198ad7d35c7f36164b is 50, key is test_row_0/C:col10/1731133051881/Put/seqid=0 2024-11-09T06:17:33,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133113053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:33,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T06:17:33,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:33,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:33,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:33,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:33,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:33,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742295_1471 (size=12001) 2024-11-09T06:17:33,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T06:17:33,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:33,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:33,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:33,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:33,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:33,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:33,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T06:17:33,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133113273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:33,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T06:17:33,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:33,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:33,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:33,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:33,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:33,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/7d8651ef67b240198ad7d35c7f36164b 2024-11-09T06:17:33,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/2be67310c49f499ea2ab7f4a2eb5ea84 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/2be67310c49f499ea2ab7f4a2eb5ea84 2024-11-09T06:17:33,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/2be67310c49f499ea2ab7f4a2eb5ea84, entries=150, sequenceid=107, filesize=11.7 K 2024-11-09T06:17:33,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/28a60559a0f54d2c9b24be225c3efe8d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/28a60559a0f54d2c9b24be225c3efe8d 2024-11-09T06:17:33,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/28a60559a0f54d2c9b24be225c3efe8d, entries=150, sequenceid=107, filesize=11.7 K 2024-11-09T06:17:33,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/7d8651ef67b240198ad7d35c7f36164b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7d8651ef67b240198ad7d35c7f36164b 2024-11-09T06:17:33,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7d8651ef67b240198ad7d35c7f36164b, entries=150, sequenceid=107, filesize=11.7 K 2024-11-09T06:17:33,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f15639486d79cf453f8c760f89863e07 in 969ms, sequenceid=107, compaction requested=true 2024-11-09T06:17:33,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:33,515 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:33,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:33,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:33,516 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:33,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:33,517 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:33,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:33,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:33,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:33,517 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/A is initiating minor compaction (all files) 2024-11-09T06:17:33,517 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/A in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:33,517 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/044e2222055c4703a2a216c7e6a076ec, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/555ee8121df2464cac3437c89ffe5e5e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/2be67310c49f499ea2ab7f4a2eb5ea84] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=35.3 K 2024-11-09T06:17:33,517 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:33,517 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 044e2222055c4703a2a216c7e6a076ec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731133050586 2024-11-09T06:17:33,518 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 555ee8121df2464cac3437c89ffe5e5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731133050711 2024-11-09T06:17:33,518 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/B is initiating minor compaction (all files) 2024-11-09T06:17:33,518 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/B in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:33,518 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/2a3f4b7de6364c5594a63a73ca4d5874, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/05b6c7ef536044f39cfeccdf5c4015a2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/28a60559a0f54d2c9b24be225c3efe8d] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=35.3 K 2024-11-09T06:17:33,518 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2be67310c49f499ea2ab7f4a2eb5ea84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731133051849 2024-11-09T06:17:33,519 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a3f4b7de6364c5594a63a73ca4d5874, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731133050586 2024-11-09T06:17:33,520 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05b6c7ef536044f39cfeccdf5c4015a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731133050711 2024-11-09T06:17:33,520 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a60559a0f54d2c9b24be225c3efe8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731133051849 2024-11-09T06:17:33,534 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#A#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:33,535 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/316531d471da411d96259a287f38359d is 50, key is test_row_0/A:col10/1731133051881/Put/seqid=0 2024-11-09T06:17:33,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T06:17:33,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:33,539 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-09T06:17:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:33,551 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:33,551 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/a65002775b8d48cc85ea18c5ae911f65 is 50, key is test_row_0/B:col10/1731133051881/Put/seqid=0 2024-11-09T06:17:33,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/8abf3c2bc4c2476895a78edd66e5b011 is 50, key is test_row_0/A:col10/1731133052932/Put/seqid=0 2024-11-09T06:17:33,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742296_1472 (size=12104) 2024-11-09T06:17:33,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742297_1473 (size=12104) 2024-11-09T06:17:33,602 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/a65002775b8d48cc85ea18c5ae911f65 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a65002775b8d48cc85ea18c5ae911f65 2024-11-09T06:17:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:33,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:33,607 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/B of f15639486d79cf453f8c760f89863e07 into a65002775b8d48cc85ea18c5ae911f65(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:33,607 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:33,607 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/B, priority=13, startTime=1731133053516; duration=0sec 2024-11-09T06:17:33,608 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:33,608 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:B 2024-11-09T06:17:33,608 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:33,609 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:33,609 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/C is initiating minor compaction (all files) 2024-11-09T06:17:33,609 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/C in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:33,609 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/792ac5c4a00448fca2c0bd35c16b3c16, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/ab6fc06aeef44b91a76165c850ae11d4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7d8651ef67b240198ad7d35c7f36164b] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=35.3 K 2024-11-09T06:17:33,609 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 792ac5c4a00448fca2c0bd35c16b3c16, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731133050586 2024-11-09T06:17:33,610 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab6fc06aeef44b91a76165c850ae11d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731133050711 2024-11-09T06:17:33,611 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d8651ef67b240198ad7d35c7f36164b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731133051849 2024-11-09T06:17:33,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742298_1474 (size=12151) 2024-11-09T06:17:33,625 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/8abf3c2bc4c2476895a78edd66e5b011 2024-11-09T06:17:33,630 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:33,630 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/1aac2638a2574aeea247de0a1c9562e7 is 50, key is test_row_0/C:col10/1731133051881/Put/seqid=0 2024-11-09T06:17:33,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/f16e4d2d6b464ca4b6fc0a059fca92f3 is 50, key is test_row_0/B:col10/1731133052932/Put/seqid=0 2024-11-09T06:17:33,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742299_1475 (size=12104) 2024-11-09T06:17:33,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742300_1476 (size=12151) 2024-11-09T06:17:33,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133113683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:33,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:33,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:33,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T06:17:33,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133113803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:34,003 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/316531d471da411d96259a287f38359d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/316531d471da411d96259a287f38359d 2024-11-09T06:17:34,009 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/A of f15639486d79cf453f8c760f89863e07 into 316531d471da411d96259a287f38359d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:34,009 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:34,009 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/A, priority=13, startTime=1731133053515; duration=0sec 2024-11-09T06:17:34,009 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:34,009 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:A 2024-11-09T06:17:34,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133114013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:34,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:34,048 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/1aac2638a2574aeea247de0a1c9562e7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/1aac2638a2574aeea247de0a1c9562e7 2024-11-09T06:17:34,057 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/C of f15639486d79cf453f8c760f89863e07 into 1aac2638a2574aeea247de0a1c9562e7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:34,057 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:34,057 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/C, priority=13, startTime=1731133053517; duration=0sec 2024-11-09T06:17:34,057 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:34,057 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:C 2024-11-09T06:17:34,095 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/f16e4d2d6b464ca4b6fc0a059fca92f3 2024-11-09T06:17:34,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/05a92dbd60c04650acea7184b4e7bead is 50, key is test_row_0/C:col10/1731133052932/Put/seqid=0 2024-11-09T06:17:34,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742301_1477 (size=12151) 2024-11-09T06:17:34,161 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/05a92dbd60c04650acea7184b4e7bead 2024-11-09T06:17:34,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/8abf3c2bc4c2476895a78edd66e5b011 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8abf3c2bc4c2476895a78edd66e5b011 2024-11-09T06:17:34,176 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8abf3c2bc4c2476895a78edd66e5b011, entries=150, sequenceid=143, filesize=11.9 K 2024-11-09T06:17:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/f16e4d2d6b464ca4b6fc0a059fca92f3 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f16e4d2d6b464ca4b6fc0a059fca92f3 2024-11-09T06:17:34,200 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f16e4d2d6b464ca4b6fc0a059fca92f3, entries=150, sequenceid=143, filesize=11.9 K 2024-11-09T06:17:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/05a92dbd60c04650acea7184b4e7bead as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/05a92dbd60c04650acea7184b4e7bead 2024-11-09T06:17:34,235 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/05a92dbd60c04650acea7184b4e7bead, entries=150, sequenceid=143, filesize=11.9 K 2024-11-09T06:17:34,236 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for f15639486d79cf453f8c760f89863e07 in 697ms, sequenceid=143, compaction requested=false 2024-11-09T06:17:34,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:34,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:34,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-09T06:17:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-09T06:17:34,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-09T06:17:34,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-11-09T06:17:34,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.6420 sec 2024-11-09T06:17:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:34,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T06:17:34,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:34,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:34,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:34,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:34,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:34,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:34,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/3e700e6b26154cc7b1bb1307b956b6a2 is 50, key is test_row_0/A:col10/1731133053680/Put/seqid=0 2024-11-09T06:17:34,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742302_1478 (size=12147) 2024-11-09T06:17:34,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/3e700e6b26154cc7b1bb1307b956b6a2 2024-11-09T06:17:34,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/0d18f3832c0c4404add338c26f9ad522 is 50, key is test_row_0/B:col10/1731133053680/Put/seqid=0 2024-11-09T06:17:34,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742303_1479 (size=9757) 2024-11-09T06:17:34,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/0d18f3832c0c4404add338c26f9ad522 2024-11-09T06:17:34,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/d05ee21627eb413d8fbab153be9aee8c is 50, key is test_row_0/C:col10/1731133053680/Put/seqid=0 2024-11-09T06:17:34,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742304_1480 (size=9757) 2024-11-09T06:17:34,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/d05ee21627eb413d8fbab153be9aee8c 2024-11-09T06:17:34,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/3e700e6b26154cc7b1bb1307b956b6a2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3e700e6b26154cc7b1bb1307b956b6a2 2024-11-09T06:17:34,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3e700e6b26154cc7b1bb1307b956b6a2, entries=150, sequenceid=157, filesize=11.9 K 2024-11-09T06:17:34,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/0d18f3832c0c4404add338c26f9ad522 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/0d18f3832c0c4404add338c26f9ad522 2024-11-09T06:17:34,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/0d18f3832c0c4404add338c26f9ad522, entries=100, sequenceid=157, filesize=9.5 K 2024-11-09T06:17:34,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/d05ee21627eb413d8fbab153be9aee8c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d05ee21627eb413d8fbab153be9aee8c 2024-11-09T06:17:34,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d05ee21627eb413d8fbab153be9aee8c, entries=100, sequenceid=157, filesize=9.5 K 2024-11-09T06:17:34,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=93.93 KB/96180 for f15639486d79cf453f8c760f89863e07 in 154ms, sequenceid=157, compaction requested=true 2024-11-09T06:17:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:34,496 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:34,496 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:34,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:34,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:34,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:34,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:34,501 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34012 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:34,501 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/B is initiating minor compaction (all files) 2024-11-09T06:17:34,501 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/B in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:34,501 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a65002775b8d48cc85ea18c5ae911f65, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f16e4d2d6b464ca4b6fc0a059fca92f3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/0d18f3832c0c4404add338c26f9ad522] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=33.2 K 2024-11-09T06:17:34,502 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36402 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:34,502 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/A is initiating minor compaction (all files) 2024-11-09T06:17:34,502 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/A in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:34,502 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/316531d471da411d96259a287f38359d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8abf3c2bc4c2476895a78edd66e5b011, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3e700e6b26154cc7b1bb1307b956b6a2] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=35.5 K 2024-11-09T06:17:34,502 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a65002775b8d48cc85ea18c5ae911f65, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731133051849 2024-11-09T06:17:34,502 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 316531d471da411d96259a287f38359d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731133051849 2024-11-09T06:17:34,502 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting f16e4d2d6b464ca4b6fc0a059fca92f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731133052907 2024-11-09T06:17:34,503 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8abf3c2bc4c2476895a78edd66e5b011, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731133052907 2024-11-09T06:17:34,503 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d18f3832c0c4404add338c26f9ad522, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731133053680 2024-11-09T06:17:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:34,503 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e700e6b26154cc7b1bb1307b956b6a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731133053673 2024-11-09T06:17:34,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T06:17:34,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:34,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:34,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:34,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:34,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:34,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:34,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/c0194c091f83421d8f653250942a7957 is 50, key is test_row_0/A:col10/1731133054502/Put/seqid=0 2024-11-09T06:17:34,518 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#A#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:34,519 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/b82b2f1fdf6741439a150b64609abb77 is 50, key is test_row_0/A:col10/1731133053680/Put/seqid=0 2024-11-09T06:17:34,535 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:34,535 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/60845caf53f74d279a9a6442e6c4bb87 is 50, key is test_row_0/B:col10/1731133053680/Put/seqid=0 2024-11-09T06:17:34,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742305_1481 (size=14541) 2024-11-09T06:17:34,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742306_1482 (size=12357) 2024-11-09T06:17:34,601 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/b82b2f1fdf6741439a150b64609abb77 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b82b2f1fdf6741439a150b64609abb77 2024-11-09T06:17:34,607 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/A of f15639486d79cf453f8c760f89863e07 into b82b2f1fdf6741439a150b64609abb77(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:34,607 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:34,607 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/A, priority=13, startTime=1731133054496; duration=0sec 2024-11-09T06:17:34,608 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:34,608 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:A 2024-11-09T06:17:34,608 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:34,609 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34012 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:34,609 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/C is initiating minor compaction (all files) 2024-11-09T06:17:34,610 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/C in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:34,612 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/1aac2638a2574aeea247de0a1c9562e7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/05a92dbd60c04650acea7184b4e7bead, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d05ee21627eb413d8fbab153be9aee8c] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=33.2 K 2024-11-09T06:17:34,613 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aac2638a2574aeea247de0a1c9562e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731133051849 2024-11-09T06:17:34,615 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05a92dbd60c04650acea7184b4e7bead, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731133052907 2024-11-09T06:17:34,616 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d05ee21627eb413d8fbab153be9aee8c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731133053680 2024-11-09T06:17:34,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742307_1483 (size=12357) 2024-11-09T06:17:34,627 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:34,628 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/443beb14cd9c4a54a6c588011e5be92a is 50, key is test_row_0/C:col10/1731133053680/Put/seqid=0 2024-11-09T06:17:34,632 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/60845caf53f74d279a9a6442e6c4bb87 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/60845caf53f74d279a9a6442e6c4bb87 2024-11-09T06:17:34,637 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/B of f15639486d79cf453f8c760f89863e07 into 60845caf53f74d279a9a6442e6c4bb87(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:34,637 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:34,638 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/B, priority=13, startTime=1731133054496; duration=0sec 2024-11-09T06:17:34,638 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:34,638 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:B 2024-11-09T06:17:34,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133114651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:34,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742308_1484 (size=12357) 2024-11-09T06:17:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T06:17:34,753 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:34,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-09T06:17:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T06:17:34,756 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:34,756 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:34,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:34,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133114773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T06:17:34,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T06:17:34,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:34,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:34,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:34,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:34,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:34,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133114963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:34,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/c0194c091f83421d8f653250942a7957 2024-11-09T06:17:34,971 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in f15639486d79cf453f8c760f89863e07 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:34,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133114963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:34,972 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in f15639486d79cf453f8c760f89863e07 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:34,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133114973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:34,978 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in f15639486d79cf453f8c760f89863e07 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:34,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133114983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:34,990 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in f15639486d79cf453f8c760f89863e07 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:34,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:35,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/f24850ccdbd24a5fb4d7995770a9aa4b is 50, key is test_row_0/B:col10/1731133054502/Put/seqid=0 2024-11-09T06:17:35,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133114994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:35,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:35,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:35,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:35,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742309_1485 (size=12151) 2024-11-09T06:17:35,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/f24850ccdbd24a5fb4d7995770a9aa4b 2024-11-09T06:17:35,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/1f46d07274f240a29b504b482a3e859b is 50, key is test_row_0/C:col10/1731133054502/Put/seqid=0 2024-11-09T06:17:35,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T06:17:35,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:35,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:35,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:35,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T06:17:35,088 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/443beb14cd9c4a54a6c588011e5be92a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/443beb14cd9c4a54a6c588011e5be92a 2024-11-09T06:17:35,100 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/C of f15639486d79cf453f8c760f89863e07 into 443beb14cd9c4a54a6c588011e5be92a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:35,100 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:35,100 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/C, priority=13, startTime=1731133054497; duration=0sec 2024-11-09T06:17:35,100 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:35,100 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:C 2024-11-09T06:17:35,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742310_1486 (size=12151) 2024-11-09T06:17:35,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T06:17:35,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:35,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:35,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:35,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133115312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:35,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T06:17:35,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:35,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:35,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:35,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T06:17:35,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/1f46d07274f240a29b504b482a3e859b 2024-11-09T06:17:35,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/c0194c091f83421d8f653250942a7957 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c0194c091f83421d8f653250942a7957 2024-11-09T06:17:35,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T06:17:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:35,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:35,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:35,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:35,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c0194c091f83421d8f653250942a7957, entries=200, sequenceid=175, filesize=14.2 K 2024-11-09T06:17:35,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/f24850ccdbd24a5fb4d7995770a9aa4b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f24850ccdbd24a5fb4d7995770a9aa4b 2024-11-09T06:17:35,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f24850ccdbd24a5fb4d7995770a9aa4b, entries=150, sequenceid=175, filesize=11.9 K 2024-11-09T06:17:35,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/1f46d07274f240a29b504b482a3e859b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/1f46d07274f240a29b504b482a3e859b 2024-11-09T06:17:35,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/1f46d07274f240a29b504b482a3e859b, entries=150, sequenceid=175, filesize=11.9 K 2024-11-09T06:17:35,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f15639486d79cf453f8c760f89863e07 in 1056ms, sequenceid=175, compaction requested=false 2024-11-09T06:17:35,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:35,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T06:17:35,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:35,689 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T06:17:35,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:35,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:35,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:35,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:35,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:35,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:35,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/86daaf2e91704bdf84d1a07b90398711 is 50, key is test_row_0/A:col10/1731133054638/Put/seqid=0 2024-11-09T06:17:35,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742311_1487 (size=12151) 2024-11-09T06:17:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:35,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T06:17:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133115990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:36,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133116103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:36,158 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/86daaf2e91704bdf84d1a07b90398711 2024-11-09T06:17:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/cd973cb3b6854c65be18e5184bceef8d is 50, key is test_row_0/B:col10/1731133054638/Put/seqid=0 2024-11-09T06:17:36,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742312_1488 (size=12151) 2024-11-09T06:17:36,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133116312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:36,581 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/cd973cb3b6854c65be18e5184bceef8d 2024-11-09T06:17:36,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/d2b946380e574a44a1a52ba4439052fc is 50, key is test_row_0/C:col10/1731133054638/Put/seqid=0 2024-11-09T06:17:36,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742313_1489 (size=12151) 2024-11-09T06:17:36,611 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/d2b946380e574a44a1a52ba4439052fc 2024-11-09T06:17:36,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/86daaf2e91704bdf84d1a07b90398711 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/86daaf2e91704bdf84d1a07b90398711 2024-11-09T06:17:36,621 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/86daaf2e91704bdf84d1a07b90398711, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T06:17:36,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/cd973cb3b6854c65be18e5184bceef8d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/cd973cb3b6854c65be18e5184bceef8d 2024-11-09T06:17:36,625 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/cd973cb3b6854c65be18e5184bceef8d, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T06:17:36,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/d2b946380e574a44a1a52ba4439052fc as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d2b946380e574a44a1a52ba4439052fc 2024-11-09T06:17:36,635 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d2b946380e574a44a1a52ba4439052fc, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T06:17:36,636 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f15639486d79cf453f8c760f89863e07 in 947ms, sequenceid=192, compaction requested=true 2024-11-09T06:17:36,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:36,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:36,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-09T06:17:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-09T06:17:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:36,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T06:17:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:36,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-09T06:17:36,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-09T06:17:36,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8880 sec 2024-11-09T06:17:36,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d07d94ac22994030922904102ae0731d is 50, key is test_row_0/A:col10/1731133056636/Put/seqid=0 2024-11-09T06:17:36,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742314_1490 (size=14541) 2024-11-09T06:17:36,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d07d94ac22994030922904102ae0731d 2024-11-09T06:17:36,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/3a6d85250d254373a533246c03c84327 is 50, key is test_row_0/B:col10/1731133056636/Put/seqid=0 2024-11-09T06:17:36,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742315_1491 (size=12151) 2024-11-09T06:17:36,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/3a6d85250d254373a533246c03c84327 2024-11-09T06:17:36,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/9b46174592f3438b8059b956368c0bb5 is 50, key is test_row_0/C:col10/1731133056636/Put/seqid=0 2024-11-09T06:17:36,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133116726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:36,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742316_1492 (size=12151) 2024-11-09T06:17:36,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133116843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:36,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T06:17:36,903 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:36,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-09T06:17:36,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T06:17:36,908 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:36,908 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:36,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T06:17:37,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T06:17:37,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:37,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:37,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:37,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:37,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:37,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133117062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:37,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/9b46174592f3438b8059b956368c0bb5 2024-11-09T06:17:37,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d07d94ac22994030922904102ae0731d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d07d94ac22994030922904102ae0731d 2024-11-09T06:17:37,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d07d94ac22994030922904102ae0731d, entries=200, sequenceid=211, filesize=14.2 K 2024-11-09T06:17:37,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/3a6d85250d254373a533246c03c84327 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3a6d85250d254373a533246c03c84327 2024-11-09T06:17:37,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3a6d85250d254373a533246c03c84327, entries=150, sequenceid=211, filesize=11.9 K 2024-11-09T06:17:37,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/9b46174592f3438b8059b956368c0bb5 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9b46174592f3438b8059b956368c0bb5 2024-11-09T06:17:37,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9b46174592f3438b8059b956368c0bb5, entries=150, sequenceid=211, filesize=11.9 K 2024-11-09T06:17:37,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f15639486d79cf453f8c760f89863e07 in 537ms, sequenceid=211, compaction requested=true 2024-11-09T06:17:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:37,176 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:37,176 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:37,178 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:37,179 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53590 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:37,179 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/A is initiating minor compaction (all files) 2024-11-09T06:17:37,179 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/B is initiating minor compaction (all files) 2024-11-09T06:17:37,179 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/B in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:37,179 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/A in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:37,179 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/60845caf53f74d279a9a6442e6c4bb87, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f24850ccdbd24a5fb4d7995770a9aa4b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/cd973cb3b6854c65be18e5184bceef8d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3a6d85250d254373a533246c03c84327] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=47.7 K 2024-11-09T06:17:37,179 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b82b2f1fdf6741439a150b64609abb77, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c0194c091f83421d8f653250942a7957, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/86daaf2e91704bdf84d1a07b90398711, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d07d94ac22994030922904102ae0731d] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=52.3 K 2024-11-09T06:17:37,180 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b82b2f1fdf6741439a150b64609abb77, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731133052919 2024-11-09T06:17:37,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:37,180 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 60845caf53f74d279a9a6442e6c4bb87, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731133052919 2024-11-09T06:17:37,183 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting f24850ccdbd24a5fb4d7995770a9aa4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731133054442 2024-11-09T06:17:37,183 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0194c091f83421d8f653250942a7957, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731133054442 2024-11-09T06:17:37,184 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86daaf2e91704bdf84d1a07b90398711, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731133054610 2024-11-09T06:17:37,185 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting cd973cb3b6854c65be18e5184bceef8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731133054610 2024-11-09T06:17:37,185 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d07d94ac22994030922904102ae0731d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731133055920 2024-11-09T06:17:37,186 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a6d85250d254373a533246c03c84327, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731133055976 2024-11-09T06:17:37,208 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#A#compaction#416 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:37,208 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/6b9aaad708704b61910da9009a33034a is 50, key is test_row_0/A:col10/1731133056636/Put/seqid=0 2024-11-09T06:17:37,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T06:17:37,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:37,214 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:17:37,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:37,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:37,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:37,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:37,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:37,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T06:17:37,224 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#B#compaction#417 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:37,225 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/473a04af862943b0a4a5aef41e0f86b7 is 50, key is test_row_0/B:col10/1731133056636/Put/seqid=0 2024-11-09T06:17:37,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/e6f27f9acca448299507f690cdebe87d is 50, key is test_row_0/A:col10/1731133056650/Put/seqid=0 2024-11-09T06:17:37,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742317_1493 (size=12493) 2024-11-09T06:17:37,288 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/6b9aaad708704b61910da9009a33034a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/6b9aaad708704b61910da9009a33034a 2024-11-09T06:17:37,298 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f15639486d79cf453f8c760f89863e07/A of f15639486d79cf453f8c760f89863e07 into 6b9aaad708704b61910da9009a33034a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:37,298 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:37,298 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/A, priority=12, startTime=1731133057176; duration=0sec 2024-11-09T06:17:37,298 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:37,298 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:A 2024-11-09T06:17:37,298 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:37,300 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:37,300 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/C is initiating minor compaction (all files) 2024-11-09T06:17:37,300 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/C in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:37,300 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/443beb14cd9c4a54a6c588011e5be92a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/1f46d07274f240a29b504b482a3e859b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d2b946380e574a44a1a52ba4439052fc, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9b46174592f3438b8059b956368c0bb5] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=47.7 K 2024-11-09T06:17:37,300 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 443beb14cd9c4a54a6c588011e5be92a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731133052919 2024-11-09T06:17:37,301 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f46d07274f240a29b504b482a3e859b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731133054442 2024-11-09T06:17:37,301 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2b946380e574a44a1a52ba4439052fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731133054610 2024-11-09T06:17:37,301 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b46174592f3438b8059b956368c0bb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731133055976 2024-11-09T06:17:37,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742318_1494 (size=12493) 2024-11-09T06:17:37,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742319_1495 (size=12151) 2024-11-09T06:17:37,323 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/e6f27f9acca448299507f690cdebe87d 2024-11-09T06:17:37,335 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#C#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:37,335 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/078551e56a4d478daa00f736d3395397 is 50, key is test_row_0/C:col10/1731133056636/Put/seqid=0 2024-11-09T06:17:37,338 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/473a04af862943b0a4a5aef41e0f86b7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/473a04af862943b0a4a5aef41e0f86b7 2024-11-09T06:17:37,350 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f15639486d79cf453f8c760f89863e07/B of f15639486d79cf453f8c760f89863e07 into 473a04af862943b0a4a5aef41e0f86b7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:37,350 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:37,350 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/B, priority=12, startTime=1731133057176; duration=0sec 2024-11-09T06:17:37,350 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:37,350 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:B 2024-11-09T06:17:37,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/f4d37c3ee6094b87aed326fa5d289356 is 50, key is test_row_0/B:col10/1731133056650/Put/seqid=0 2024-11-09T06:17:37,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:37,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742321_1497 (size=12151) 2024-11-09T06:17:37,393 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/f4d37c3ee6094b87aed326fa5d289356 2024-11-09T06:17:37,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742320_1496 (size=12493) 2024-11-09T06:17:37,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/25de12a932c944179ece27131fed1d96 is 50, key is test_row_0/C:col10/1731133056650/Put/seqid=0 2024-11-09T06:17:37,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742322_1498 (size=12151) 2024-11-09T06:17:37,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133117511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:37,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:37,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:37,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T06:17:37,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133117622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:37,797 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/078551e56a4d478daa00f736d3395397 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/078551e56a4d478daa00f736d3395397 2024-11-09T06:17:37,802 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f15639486d79cf453f8c760f89863e07/C of f15639486d79cf453f8c760f89863e07 into 078551e56a4d478daa00f736d3395397(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:37,802 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:37,803 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/C, priority=12, startTime=1731133057176; duration=0sec 2024-11-09T06:17:37,803 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:37,803 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:C 2024-11-09T06:17:37,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133117832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:37,850 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/25de12a932c944179ece27131fed1d96 2024-11-09T06:17:37,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/e6f27f9acca448299507f690cdebe87d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/e6f27f9acca448299507f690cdebe87d 2024-11-09T06:17:37,866 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/e6f27f9acca448299507f690cdebe87d, entries=150, sequenceid=224, filesize=11.9 K 2024-11-09T06:17:37,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/f4d37c3ee6094b87aed326fa5d289356 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f4d37c3ee6094b87aed326fa5d289356 2024-11-09T06:17:37,879 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f4d37c3ee6094b87aed326fa5d289356, entries=150, sequenceid=224, filesize=11.9 K 2024-11-09T06:17:37,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/25de12a932c944179ece27131fed1d96 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/25de12a932c944179ece27131fed1d96 2024-11-09T06:17:37,894 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/25de12a932c944179ece27131fed1d96, entries=150, sequenceid=224, filesize=11.9 K 2024-11-09T06:17:37,895 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f15639486d79cf453f8c760f89863e07 in 681ms, sequenceid=224, compaction requested=false 2024-11-09T06:17:37,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:37,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:37,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-09T06:17:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-09T06:17:37,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-09T06:17:37,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.0020 sec 2024-11-09T06:17:37,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 991 msec 2024-11-09T06:17:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T06:17:38,043 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:38,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-09T06:17:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T06:17:38,045 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:38,046 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:38,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T06:17:38,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:17:38,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:38,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:38,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:38,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:38,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:38,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:38,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/84c94d35389d448592291d756cc59b1e is 50, key is test_row_0/A:col10/1731133058160/Put/seqid=0 2024-11-09T06:17:38,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:38,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:38,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742323_1499 (size=14541) 2024-11-09T06:17:38,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/84c94d35389d448592291d756cc59b1e 2024-11-09T06:17:38,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/033cd170d07145c09871681d0eb4d894 is 50, key is test_row_0/B:col10/1731133058160/Put/seqid=0 2024-11-09T06:17:38,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742324_1500 (size=12151) 2024-11-09T06:17:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133118293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:38,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T06:17:38,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133118422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:38,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:38,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:38,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133118643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:38,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T06:17:38,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/033cd170d07145c09871681d0eb4d894 2024-11-09T06:17:38,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/9e16e2779a9445fc8112da7bd6755649 is 50, key is test_row_0/C:col10/1731133058160/Put/seqid=0 2024-11-09T06:17:38,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742325_1501 (size=12151) 2024-11-09T06:17:38,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/9e16e2779a9445fc8112da7bd6755649 2024-11-09T06:17:38,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/84c94d35389d448592291d756cc59b1e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/84c94d35389d448592291d756cc59b1e 2024-11-09T06:17:38,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/84c94d35389d448592291d756cc59b1e, entries=200, sequenceid=247, filesize=14.2 K 2024-11-09T06:17:38,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/033cd170d07145c09871681d0eb4d894 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/033cd170d07145c09871681d0eb4d894 2024-11-09T06:17:38,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/033cd170d07145c09871681d0eb4d894, entries=150, sequenceid=247, filesize=11.9 K 2024-11-09T06:17:38,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/9e16e2779a9445fc8112da7bd6755649 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9e16e2779a9445fc8112da7bd6755649 2024-11-09T06:17:38,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:38,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9e16e2779a9445fc8112da7bd6755649, entries=150, sequenceid=247, filesize=11.9 K 2024-11-09T06:17:38,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f15639486d79cf453f8c760f89863e07 in 672ms, sequenceid=247, compaction requested=true 2024-11-09T06:17:38,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:38,833 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:38,835 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:38,835 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/A is initiating minor compaction (all files) 2024-11-09T06:17:38,835 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/A in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,835 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/6b9aaad708704b61910da9009a33034a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/e6f27f9acca448299507f690cdebe87d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/84c94d35389d448592291d756cc59b1e] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=38.3 K 2024-11-09T06:17:38,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:38,836 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:38,836 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b9aaad708704b61910da9009a33034a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731133055976 2024-11-09T06:17:38,837 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f27f9acca448299507f690cdebe87d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731133056650 2024-11-09T06:17:38,838 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:38,838 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/B is initiating minor compaction (all files) 2024-11-09T06:17:38,838 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/B in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,838 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/473a04af862943b0a4a5aef41e0f86b7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f4d37c3ee6094b87aed326fa5d289356, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/033cd170d07145c09871681d0eb4d894] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=35.9 K 2024-11-09T06:17:38,838 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84c94d35389d448592291d756cc59b1e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731133057479 2024-11-09T06:17:38,839 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 473a04af862943b0a4a5aef41e0f86b7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731133055976 2024-11-09T06:17:38,840 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d37c3ee6094b87aed326fa5d289356, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731133056650 2024-11-09T06:17:38,840 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 033cd170d07145c09871681d0eb4d894, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731133057479 2024-11-09T06:17:38,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:38,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:38,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:38,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:38,858 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#A#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:38,860 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/b5ade32801c2417297ea3445c0024d13 is 50, key is test_row_0/A:col10/1731133058160/Put/seqid=0 2024-11-09T06:17:38,868 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:38,868 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/f6e2a164eec04bda894949ff557e28b0 is 50, key is test_row_0/B:col10/1731133058160/Put/seqid=0 2024-11-09T06:17:38,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742326_1502 (size=12595) 2024-11-09T06:17:38,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742327_1503 (size=12595) 2024-11-09T06:17:38,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:17:38,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:38,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:38,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:38,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:38,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:38,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:38,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:38,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:38,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:38,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:38,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/dfe8fdee7bd64d409c096933c646aeee is 50, key is test_row_0/A:col10/1731133058289/Put/seqid=0 2024-11-09T06:17:38,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742328_1504 (size=14691) 2024-11-09T06:17:39,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133119023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133119032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133119037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133119041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133119042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:39,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:39,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133119153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133119153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133119162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133119162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133119162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T06:17:39,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:39,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:39,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,336 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/b5ade32801c2417297ea3445c0024d13 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b5ade32801c2417297ea3445c0024d13 2024-11-09T06:17:39,347 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/A of f15639486d79cf453f8c760f89863e07 into b5ade32801c2417297ea3445c0024d13(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:39,347 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:39,347 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/A, priority=13, startTime=1731133058833; duration=0sec 2024-11-09T06:17:39,347 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:39,347 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:A 2024-11-09T06:17:39,347 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:39,353 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:39,354 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/C is initiating minor compaction (all files) 2024-11-09T06:17:39,354 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/C in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,354 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/078551e56a4d478daa00f736d3395397, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/25de12a932c944179ece27131fed1d96, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9e16e2779a9445fc8112da7bd6755649] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=35.9 K 2024-11-09T06:17:39,354 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 078551e56a4d478daa00f736d3395397, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731133055976 2024-11-09T06:17:39,355 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25de12a932c944179ece27131fed1d96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731133056650 2024-11-09T06:17:39,355 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e16e2779a9445fc8112da7bd6755649, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731133057479 2024-11-09T06:17:39,364 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/f6e2a164eec04bda894949ff557e28b0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f6e2a164eec04bda894949ff557e28b0 2024-11-09T06:17:39,369 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/B of f15639486d79cf453f8c760f89863e07 into f6e2a164eec04bda894949ff557e28b0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:39,369 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:39,369 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/B, priority=13, startTime=1731133058836; duration=0sec 2024-11-09T06:17:39,369 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:39,370 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:B 2024-11-09T06:17:39,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133119373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133119376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,391 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#C#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:39,391 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/033b069606ab4768ac2a202e6586a22f is 50, key is test_row_0/C:col10/1731133058160/Put/seqid=0 2024-11-09T06:17:39,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/dfe8fdee7bd64d409c096933c646aeee 2024-11-09T06:17:39,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133119383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133119384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133119385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742329_1505 (size=12595) 2024-11-09T06:17:39,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/d8607dbcbb06405e9c2e75cc36e0d50b is 50, key is test_row_0/B:col10/1731133058289/Put/seqid=0 2024-11-09T06:17:39,441 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/033b069606ab4768ac2a202e6586a22f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/033b069606ab4768ac2a202e6586a22f 2024-11-09T06:17:39,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:39,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:39,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,447 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/C of f15639486d79cf453f8c760f89863e07 into 033b069606ab4768ac2a202e6586a22f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:39,447 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:39,447 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/C, priority=13, startTime=1731133058850; duration=0sec 2024-11-09T06:17:39,448 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:39,448 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:C 2024-11-09T06:17:39,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742330_1506 (size=12251) 2024-11-09T06:17:39,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/d8607dbcbb06405e9c2e75cc36e0d50b 2024-11-09T06:17:39,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/c7292ceb7b904cb1969e71f34a1748da is 50, key is test_row_0/C:col10/1731133058289/Put/seqid=0 2024-11-09T06:17:39,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742331_1507 (size=12251) 2024-11-09T06:17:39,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:39,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:39,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133119682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133119682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133119702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133119702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133119702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:39,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:39,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:39,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/c7292ceb7b904cb1969e71f34a1748da 2024-11-09T06:17:39,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/dfe8fdee7bd64d409c096933c646aeee as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/dfe8fdee7bd64d409c096933c646aeee 2024-11-09T06:17:39,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/dfe8fdee7bd64d409c096933c646aeee, entries=200, sequenceid=260, filesize=14.3 K 2024-11-09T06:17:39,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/d8607dbcbb06405e9c2e75cc36e0d50b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d8607dbcbb06405e9c2e75cc36e0d50b 2024-11-09T06:17:39,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d8607dbcbb06405e9c2e75cc36e0d50b, entries=150, sequenceid=260, filesize=12.0 K 2024-11-09T06:17:39,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/c7292ceb7b904cb1969e71f34a1748da as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/c7292ceb7b904cb1969e71f34a1748da 2024-11-09T06:17:39,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:39,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:39,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:39,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/c7292ceb7b904cb1969e71f34a1748da, entries=150, sequenceid=260, filesize=12.0 K 2024-11-09T06:17:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:39,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f15639486d79cf453f8c760f89863e07 in 953ms, sequenceid=260, compaction requested=false 2024-11-09T06:17:39,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:40,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T06:17:40,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:40,080 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T06:17:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:40,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/1ad048a474fa40688c7e54e15444d85d is 50, key is test_row_0/A:col10/1731133059032/Put/seqid=0 2024-11-09T06:17:40,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742332_1508 (size=12301) 2024-11-09T06:17:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T06:17:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:40,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:40,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133120224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133120225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133120230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133120232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133120233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133120342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133120342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133120353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133120353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133120353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,548 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/1ad048a474fa40688c7e54e15444d85d 2024-11-09T06:17:40,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/4d573700d26e479e8605377d1431d96d is 50, key is test_row_0/B:col10/1731133059032/Put/seqid=0 2024-11-09T06:17:40,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133120553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133120563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742333_1509 (size=12301) 2024-11-09T06:17:40,575 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/4d573700d26e479e8605377d1431d96d 2024-11-09T06:17:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133120572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133120573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133120573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/7a5980968061484ba93a03b2d5edeaed is 50, key is test_row_0/C:col10/1731133059032/Put/seqid=0 2024-11-09T06:17:40,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742334_1510 (size=12301) 2024-11-09T06:17:40,598 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/7a5980968061484ba93a03b2d5edeaed 2024-11-09T06:17:40,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/1ad048a474fa40688c7e54e15444d85d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1ad048a474fa40688c7e54e15444d85d 2024-11-09T06:17:40,607 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1ad048a474fa40688c7e54e15444d85d, entries=150, sequenceid=282, filesize=12.0 K 2024-11-09T06:17:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/4d573700d26e479e8605377d1431d96d as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/4d573700d26e479e8605377d1431d96d 2024-11-09T06:17:40,633 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/4d573700d26e479e8605377d1431d96d, entries=150, sequenceid=282, filesize=12.0 K 2024-11-09T06:17:40,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/7a5980968061484ba93a03b2d5edeaed as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7a5980968061484ba93a03b2d5edeaed 2024-11-09T06:17:40,637 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7a5980968061484ba93a03b2d5edeaed, entries=150, sequenceid=282, filesize=12.0 K 2024-11-09T06:17:40,638 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f15639486d79cf453f8c760f89863e07 in 558ms, sequenceid=282, compaction requested=true 2024-11-09T06:17:40,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:40,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:40,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-09T06:17:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-09T06:17:40,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-09T06:17:40,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5940 sec 2024-11-09T06:17:40,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.6030 sec 2024-11-09T06:17:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:40,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T06:17:40,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:40,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:40,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:40,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:40,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:40,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:40,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/c3bc4a9bab384e6b8790741206666b4a is 50, key is test_row_0/A:col10/1731133060227/Put/seqid=0 2024-11-09T06:17:40,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742335_1511 (size=14741) 2024-11-09T06:17:40,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133120925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133120925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133120926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133120927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:40,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133120932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133121042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133121042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133121053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133121053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133121053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133121253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133121253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133121263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133121264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133121264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/c3bc4a9bab384e6b8790741206666b4a 2024-11-09T06:17:41,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/93dac15778244c438ca8c57c0abe6679 is 50, key is test_row_0/B:col10/1731133060227/Put/seqid=0 2024-11-09T06:17:41,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742336_1512 (size=12301) 2024-11-09T06:17:41,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/93dac15778244c438ca8c57c0abe6679 2024-11-09T06:17:41,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/18ac09105c25440582f59f93b8daa0ad is 50, key is test_row_0/C:col10/1731133060227/Put/seqid=0 2024-11-09T06:17:41,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742337_1513 (size=12301) 2024-11-09T06:17:41,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133121573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133121573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133121582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133121582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133121582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:41,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/18ac09105c25440582f59f93b8daa0ad 2024-11-09T06:17:41,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/c3bc4a9bab384e6b8790741206666b4a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c3bc4a9bab384e6b8790741206666b4a 2024-11-09T06:17:41,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c3bc4a9bab384e6b8790741206666b4a, entries=200, sequenceid=296, filesize=14.4 K 2024-11-09T06:17:41,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/93dac15778244c438ca8c57c0abe6679 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/93dac15778244c438ca8c57c0abe6679 2024-11-09T06:17:41,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/93dac15778244c438ca8c57c0abe6679, entries=150, sequenceid=296, filesize=12.0 K 2024-11-09T06:17:41,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/18ac09105c25440582f59f93b8daa0ad as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/18ac09105c25440582f59f93b8daa0ad 2024-11-09T06:17:41,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/18ac09105c25440582f59f93b8daa0ad, entries=150, sequenceid=296, filesize=12.0 K 2024-11-09T06:17:41,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f15639486d79cf453f8c760f89863e07 in 971ms, sequenceid=296, compaction requested=true 2024-11-09T06:17:41,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:41,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:41,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:41,843 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:41,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:41,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:41,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:41,843 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:41,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:41,844 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:41,844 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/A is initiating minor compaction (all files) 2024-11-09T06:17:41,844 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/A in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:41,844 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b5ade32801c2417297ea3445c0024d13, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/dfe8fdee7bd64d409c096933c646aeee, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1ad048a474fa40688c7e54e15444d85d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c3bc4a9bab384e6b8790741206666b4a] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=53.1 K 2024-11-09T06:17:41,844 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5ade32801c2417297ea3445c0024d13, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731133057479 2024-11-09T06:17:41,845 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfe8fdee7bd64d409c096933c646aeee, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731133058229 2024-11-09T06:17:41,845 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ad048a474fa40688c7e54e15444d85d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731133059020 2024-11-09T06:17:41,845 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49448 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:41,846 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/B is initiating minor compaction (all files) 2024-11-09T06:17:41,846 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/B in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:41,846 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f6e2a164eec04bda894949ff557e28b0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d8607dbcbb06405e9c2e75cc36e0d50b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/4d573700d26e479e8605377d1431d96d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/93dac15778244c438ca8c57c0abe6679] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=48.3 K 2024-11-09T06:17:41,847 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3bc4a9bab384e6b8790741206666b4a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731133060209 2024-11-09T06:17:41,848 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting f6e2a164eec04bda894949ff557e28b0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731133057479 2024-11-09T06:17:41,848 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting d8607dbcbb06405e9c2e75cc36e0d50b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731133058229 2024-11-09T06:17:41,849 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d573700d26e479e8605377d1431d96d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731133059020 2024-11-09T06:17:41,849 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 93dac15778244c438ca8c57c0abe6679, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731133060209 2024-11-09T06:17:41,867 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#A#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:41,867 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/6e04e39d639c4e4d9b5668473b2a54bf is 50, key is test_row_0/A:col10/1731133060227/Put/seqid=0 2024-11-09T06:17:41,875 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:41,876 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/82b9b0cb201a408eaeca215545870b4a is 50, key is test_row_0/B:col10/1731133060227/Put/seqid=0 2024-11-09T06:17:41,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742338_1514 (size=12881) 2024-11-09T06:17:41,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742339_1515 (size=12881) 2024-11-09T06:17:41,942 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/82b9b0cb201a408eaeca215545870b4a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/82b9b0cb201a408eaeca215545870b4a 2024-11-09T06:17:41,952 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f15639486d79cf453f8c760f89863e07/B of f15639486d79cf453f8c760f89863e07 into 82b9b0cb201a408eaeca215545870b4a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:41,952 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:41,952 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/B, priority=12, startTime=1731133061843; duration=0sec 2024-11-09T06:17:41,952 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:41,952 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:B 2024-11-09T06:17:41,952 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:41,955 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49448 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:41,955 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/C is initiating minor compaction (all files) 2024-11-09T06:17:41,955 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/C in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:41,955 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/033b069606ab4768ac2a202e6586a22f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/c7292ceb7b904cb1969e71f34a1748da, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7a5980968061484ba93a03b2d5edeaed, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/18ac09105c25440582f59f93b8daa0ad] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=48.3 K 2024-11-09T06:17:41,956 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 033b069606ab4768ac2a202e6586a22f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731133057479 2024-11-09T06:17:41,957 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting c7292ceb7b904cb1969e71f34a1748da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731133058229 2024-11-09T06:17:41,958 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a5980968061484ba93a03b2d5edeaed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731133059020 2024-11-09T06:17:41,958 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ac09105c25440582f59f93b8daa0ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731133060209 2024-11-09T06:17:41,972 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#C#compaction#439 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:41,973 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/c39859eaf90f47ebad729656a11b5ab2 is 50, key is test_row_0/C:col10/1731133060227/Put/seqid=0 2024-11-09T06:17:41,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742340_1516 (size=12881) 2024-11-09T06:17:41,989 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/c39859eaf90f47ebad729656a11b5ab2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/c39859eaf90f47ebad729656a11b5ab2 2024-11-09T06:17:41,997 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f15639486d79cf453f8c760f89863e07/C of f15639486d79cf453f8c760f89863e07 into c39859eaf90f47ebad729656a11b5ab2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:41,997 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:41,997 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/C, priority=12, startTime=1731133061843; duration=0sec 2024-11-09T06:17:41,997 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:41,997 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:C 2024-11-09T06:17:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:42,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:17:42,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:42,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:42,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:42,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:42,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:42,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:42,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/46e8cfd2f1434495971df4217394af03 is 50, key is test_row_0/A:col10/1731133060904/Put/seqid=0 2024-11-09T06:17:42,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133122119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133122121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133122122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133122124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133122127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742341_1517 (size=12301) 2024-11-09T06:17:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T06:17:42,213 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:42,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-09T06:17:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T06:17:42,216 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:42,217 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:42,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:42,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133122233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133122242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133122242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133122252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133122252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,322 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/6e04e39d639c4e4d9b5668473b2a54bf as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/6e04e39d639c4e4d9b5668473b2a54bf 2024-11-09T06:17:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T06:17:42,327 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f15639486d79cf453f8c760f89863e07/A of f15639486d79cf453f8c760f89863e07 into 6e04e39d639c4e4d9b5668473b2a54bf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:42,327 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:42,327 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/A, priority=12, startTime=1731133061842; duration=0sec 2024-11-09T06:17:42,327 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:42,327 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:A 2024-11-09T06:17:42,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T06:17:42,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:42,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:42,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:42,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133122452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133122462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133122463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133122473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133122473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T06:17:42,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:42,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:42,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:42,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T06:17:42,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/46e8cfd2f1434495971df4217394af03 2024-11-09T06:17:42,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/3c966174be9f43fcbe64a3cd12279374 is 50, key is test_row_0/B:col10/1731133060904/Put/seqid=0 2024-11-09T06:17:42,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742342_1518 (size=12301) 2024-11-09T06:17:42,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T06:17:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:42,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133122773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133122783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133122783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133122783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133122783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:42,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:42,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:42,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T06:17:42,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:42,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:42,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:42,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T06:17:42,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/3c966174be9f43fcbe64a3cd12279374 2024-11-09T06:17:42,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T06:17:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:42,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:42,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/79c2d4ae9e02488ebcb3a48d16b18b47 is 50, key is test_row_0/C:col10/1731133060904/Put/seqid=0 2024-11-09T06:17:43,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742343_1519 (size=12301) 2024-11-09T06:17:43,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/79c2d4ae9e02488ebcb3a48d16b18b47 2024-11-09T06:17:43,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/46e8cfd2f1434495971df4217394af03 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/46e8cfd2f1434495971df4217394af03 2024-11-09T06:17:43,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/46e8cfd2f1434495971df4217394af03, entries=150, sequenceid=319, filesize=12.0 K 2024-11-09T06:17:43,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/3c966174be9f43fcbe64a3cd12279374 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3c966174be9f43fcbe64a3cd12279374 2024-11-09T06:17:43,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3c966174be9f43fcbe64a3cd12279374, entries=150, sequenceid=319, filesize=12.0 K 2024-11-09T06:17:43,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/79c2d4ae9e02488ebcb3a48d16b18b47 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/79c2d4ae9e02488ebcb3a48d16b18b47 2024-11-09T06:17:43,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/79c2d4ae9e02488ebcb3a48d16b18b47, entries=150, sequenceid=319, filesize=12.0 K 2024-11-09T06:17:43,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for f15639486d79cf453f8c760f89863e07 in 942ms, sequenceid=319, compaction requested=false 2024-11-09T06:17:43,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:43,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T06:17:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:43,144 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T06:17:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:43,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:43,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:43,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:43,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:43,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d798d6425b7f4915a958e9aac67b16eb is 50, key is test_row_0/A:col10/1731133062123/Put/seqid=0 2024-11-09T06:17:43,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742344_1520 (size=12301) 2024-11-09T06:17:43,202 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d798d6425b7f4915a958e9aac67b16eb 2024-11-09T06:17:43,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/d18d300109ab4778a8914ede5db2f0fa is 50, key is test_row_0/B:col10/1731133062123/Put/seqid=0 2024-11-09T06:17:43,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742345_1521 (size=12301) 2024-11-09T06:17:43,236 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/d18d300109ab4778a8914ede5db2f0fa 2024-11-09T06:17:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/eb9e9d3832c14f6190fac47247c1ccfb is 50, key is test_row_0/C:col10/1731133062123/Put/seqid=0 2024-11-09T06:17:43,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742346_1522 (size=12301) 2024-11-09T06:17:43,270 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/eb9e9d3832c14f6190fac47247c1ccfb 2024-11-09T06:17:43,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d798d6425b7f4915a958e9aac67b16eb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d798d6425b7f4915a958e9aac67b16eb 2024-11-09T06:17:43,292 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d798d6425b7f4915a958e9aac67b16eb, entries=150, sequenceid=331, filesize=12.0 K 2024-11-09T06:17:43,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/d18d300109ab4778a8914ede5db2f0fa as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d18d300109ab4778a8914ede5db2f0fa 2024-11-09T06:17:43,297 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d18d300109ab4778a8914ede5db2f0fa, entries=150, sequenceid=331, filesize=12.0 K 2024-11-09T06:17:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/eb9e9d3832c14f6190fac47247c1ccfb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/eb9e9d3832c14f6190fac47247c1ccfb 2024-11-09T06:17:43,304 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/eb9e9d3832c14f6190fac47247c1ccfb, entries=150, sequenceid=331, filesize=12.0 K 2024-11-09T06:17:43,305 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for f15639486d79cf453f8c760f89863e07 in 161ms, sequenceid=331, compaction requested=true 2024-11-09T06:17:43,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:43,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:43,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-09T06:17:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-09T06:17:43,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-09T06:17:43,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0910 sec 2024-11-09T06:17:43,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.1020 sec 2024-11-09T06:17:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:43,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:17:43,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:43,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:43,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:43,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:43,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:43,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:43,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/1a80cdfa494f4bdca9fe1cb48525a849 is 50, key is test_row_0/A:col10/1731133063329/Put/seqid=0 2024-11-09T06:17:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T06:17:43,353 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:43,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-09T06:17:43,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742347_1523 (size=17181) 2024-11-09T06:17:43,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/1a80cdfa494f4bdca9fe1cb48525a849 2024-11-09T06:17:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T06:17:43,358 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:43,360 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:43,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:43,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/9e726fcf655547e691e7a222ef072b99 is 50, key is test_row_0/B:col10/1731133063329/Put/seqid=0 2024-11-09T06:17:43,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742348_1524 (size=12301) 2024-11-09T06:17:43,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133063351, max timestamp=1731133063404Num uniques 859; 2024-11-09T06:17:43,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:17:43,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133063351, max timestamp=1731133063404Num uniques 859; 2024-11-09T06:17:43,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:17:43,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133063351, max timestamp=1731133063404Num uniques 859; 2024-11-09T06:17:43,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:17:43,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133123442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T06:17:43,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133123459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133123459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133123460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133123461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T06:17:43,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:43,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:43,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:43,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:43,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:43,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133123572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133123583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133123583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133123592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133123592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T06:17:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:43,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T06:17:43,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/9e726fcf655547e691e7a222ef072b99 2024-11-09T06:17:43,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/fc04ad06f2014e87877e5b7b01529f0b is 50, key is test_row_0/C:col10/1731133063329/Put/seqid=0 2024-11-09T06:17:43,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133123792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133123803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133123804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742349_1525 (size=12301) 2024-11-09T06:17:43,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/fc04ad06f2014e87877e5b7b01529f0b 2024-11-09T06:17:43,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T06:17:43,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:43,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:43,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:43,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:43,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:43,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/1a80cdfa494f4bdca9fe1cb48525a849 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1a80cdfa494f4bdca9fe1cb48525a849 2024-11-09T06:17:43,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133123812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133123812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:43,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1a80cdfa494f4bdca9fe1cb48525a849, entries=250, sequenceid=341, filesize=16.8 K 2024-11-09T06:17:43,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/9e726fcf655547e691e7a222ef072b99 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/9e726fcf655547e691e7a222ef072b99 2024-11-09T06:17:43,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/9e726fcf655547e691e7a222ef072b99, entries=150, sequenceid=341, filesize=12.0 K 2024-11-09T06:17:43,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/fc04ad06f2014e87877e5b7b01529f0b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/fc04ad06f2014e87877e5b7b01529f0b 2024-11-09T06:17:43,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/fc04ad06f2014e87877e5b7b01529f0b, entries=150, sequenceid=341, filesize=12.0 K 2024-11-09T06:17:43,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f15639486d79cf453f8c760f89863e07 in 516ms, sequenceid=341, compaction requested=true 2024-11-09T06:17:43,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:43,847 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:43,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:43,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:43,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:43,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:43,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:43,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T06:17:43,847 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:43,850 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:43,850 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/A is initiating minor compaction (all files) 2024-11-09T06:17:43,850 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/A in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:43,850 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/6e04e39d639c4e4d9b5668473b2a54bf, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/46e8cfd2f1434495971df4217394af03, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d798d6425b7f4915a958e9aac67b16eb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1a80cdfa494f4bdca9fe1cb48525a849] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=53.4 K 2024-11-09T06:17:43,851 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:43,851 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/B is initiating minor compaction (all files) 2024-11-09T06:17:43,851 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/B in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:43,851 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e04e39d639c4e4d9b5668473b2a54bf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731133060209 2024-11-09T06:17:43,851 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/82b9b0cb201a408eaeca215545870b4a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3c966174be9f43fcbe64a3cd12279374, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d18d300109ab4778a8914ede5db2f0fa, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/9e726fcf655547e691e7a222ef072b99] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=48.6 K 2024-11-09T06:17:43,851 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e8cfd2f1434495971df4217394af03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731133060904 2024-11-09T06:17:43,851 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82b9b0cb201a408eaeca215545870b4a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731133060209 2024-11-09T06:17:43,851 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting d798d6425b7f4915a958e9aac67b16eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731133062117 2024-11-09T06:17:43,852 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c966174be9f43fcbe64a3cd12279374, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731133060904 2024-11-09T06:17:43,852 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a80cdfa494f4bdca9fe1cb48525a849, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731133063307 2024-11-09T06:17:43,853 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d18d300109ab4778a8914ede5db2f0fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731133062117 2024-11-09T06:17:43,853 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e726fcf655547e691e7a222ef072b99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731133063307 2024-11-09T06:17:43,866 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#B#compaction#449 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:43,867 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/a0e348b9a70745689e2d8e5d5713ae9e is 50, key is test_row_0/B:col10/1731133063329/Put/seqid=0 2024-11-09T06:17:43,883 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#A#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:43,883 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/b04ad9b1043f4550863ceab041135c00 is 50, key is test_row_0/A:col10/1731133063329/Put/seqid=0 2024-11-09T06:17:43,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742351_1527 (size=13017) 2024-11-09T06:17:43,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742350_1526 (size=13017) 2024-11-09T06:17:43,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T06:17:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:43,974 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-09T06:17:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:43,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/88cb4f03f1f4421294abc17d539f34a0 is 50, key is test_row_0/A:col10/1731133063457/Put/seqid=0 2024-11-09T06:17:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T06:17:43,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742352_1528 (size=12301) 2024-11-09T06:17:43,992 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/88cb4f03f1f4421294abc17d539f34a0 2024-11-09T06:17:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/ad8af4ddc5c3439f96df36ed44328165 is 50, key is test_row_0/B:col10/1731133063457/Put/seqid=0 2024-11-09T06:17:44,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742353_1529 (size=12301) 2024-11-09T06:17:44,018 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/ad8af4ddc5c3439f96df36ed44328165 2024-11-09T06:17:44,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/b0b8aa6e09484821a46d1dcbf826eb33 is 50, key is test_row_0/C:col10/1731133063457/Put/seqid=0 2024-11-09T06:17:44,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742354_1530 (size=12301) 2024-11-09T06:17:44,050 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/b0b8aa6e09484821a46d1dcbf826eb33 2024-11-09T06:17:44,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/88cb4f03f1f4421294abc17d539f34a0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/88cb4f03f1f4421294abc17d539f34a0 2024-11-09T06:17:44,062 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/88cb4f03f1f4421294abc17d539f34a0, entries=150, sequenceid=378, filesize=12.0 K 2024-11-09T06:17:44,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/ad8af4ddc5c3439f96df36ed44328165 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ad8af4ddc5c3439f96df36ed44328165 2024-11-09T06:17:44,066 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ad8af4ddc5c3439f96df36ed44328165, entries=150, sequenceid=378, filesize=12.0 K 2024-11-09T06:17:44,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/b0b8aa6e09484821a46d1dcbf826eb33 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/b0b8aa6e09484821a46d1dcbf826eb33 2024-11-09T06:17:44,073 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/b0b8aa6e09484821a46d1dcbf826eb33, entries=150, sequenceid=378, filesize=12.0 K 2024-11-09T06:17:44,075 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for f15639486d79cf453f8c760f89863e07 in 101ms, sequenceid=378, compaction requested=true 2024-11-09T06:17:44,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:44,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:44,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-09T06:17:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-09T06:17:44,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-09T06:17:44,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 718 msec 2024-11-09T06:17:44,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 729 msec 2024-11-09T06:17:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:44,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:17:44,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:44,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:44,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:44,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:44,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:44,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:44,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/3ef9bf5c36bb470d94eb5415d9a508b8 is 50, key is test_row_0/A:col10/1731133064139/Put/seqid=0 2024-11-09T06:17:44,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742355_1531 (size=17181) 2024-11-09T06:17:44,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133124183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133124183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133124189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731133064180, max timestamp=1731133064200Num uniques 859; 2024-11-09T06:17:44,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:44,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731133064180, max timestamp=1731133064200Num uniques 859; 2024-11-09T06:17:44,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-09T06:17:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731133064180, max timestamp=1731133064200Num uniques 859; 2024-11-09T06:17:44,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:44,320 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/b04ad9b1043f4550863ceab041135c00 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b04ad9b1043f4550863ceab041135c00 2024-11-09T06:17:44,329 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f15639486d79cf453f8c760f89863e07/A of f15639486d79cf453f8c760f89863e07 into b04ad9b1043f4550863ceab041135c00(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:44,329 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:44,329 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/A, priority=12, startTime=1731133063847; duration=0sec 2024-11-09T06:17:44,329 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:44,329 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:A 2024-11-09T06:17:44,329 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T06:17:44,333 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62085 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T06:17:44,334 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/C is initiating minor compaction (all files) 2024-11-09T06:17:44,334 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/C in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:44,334 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/c39859eaf90f47ebad729656a11b5ab2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/79c2d4ae9e02488ebcb3a48d16b18b47, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/eb9e9d3832c14f6190fac47247c1ccfb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/fc04ad06f2014e87877e5b7b01529f0b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/b0b8aa6e09484821a46d1dcbf826eb33] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=60.6 K 2024-11-09T06:17:44,334 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting c39859eaf90f47ebad729656a11b5ab2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731133060209 2024-11-09T06:17:44,335 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 79c2d4ae9e02488ebcb3a48d16b18b47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731133060904 2024-11-09T06:17:44,335 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting eb9e9d3832c14f6190fac47247c1ccfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731133062117 2024-11-09T06:17:44,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133124313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133124313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133124318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,336 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting fc04ad06f2014e87877e5b7b01529f0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731133063307 2024-11-09T06:17:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,338 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b0b8aa6e09484821a46d1dcbf826eb33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731133063440 2024-11-09T06:17:44,339 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/a0e348b9a70745689e2d8e5d5713ae9e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a0e348b9a70745689e2d8e5d5713ae9e 2024-11-09T06:17:44,344 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f15639486d79cf453f8c760f89863e07/B of f15639486d79cf453f8c760f89863e07 into a0e348b9a70745689e2d8e5d5713ae9e(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:44,344 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:44,344 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/B, priority=12, startTime=1731133063847; duration=0sec 2024-11-09T06:17:44,344 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:44,344 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:B 2024-11-09T06:17:44,351 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#C#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:44,352 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/43c4330fdb6c43638b8bc8de9ea99809 is 50, key is test_row_0/C:col10/1731133063457/Put/seqid=0 2024-11-09T06:17:44,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133124332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133124336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742356_1532 (size=13051) 2024-11-09T06:17:44,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133124443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133124463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133124463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T06:17:44,493 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:44,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-09T06:17:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T06:17:44,496 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:44,496 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:44,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:44,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/3ef9bf5c36bb470d94eb5415d9a508b8 2024-11-09T06:17:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133124542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133124542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/236fbac84edf45f189a4c226258cf3f8 is 50, key is test_row_0/B:col10/1731133064139/Put/seqid=0 2024-11-09T06:17:44,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742357_1533 (size=12301) 2024-11-09T06:17:44,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/236fbac84edf45f189a4c226258cf3f8 2024-11-09T06:17:44,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/662d17589a5148359c5e156a2c0ff1cb is 50, key is test_row_0/C:col10/1731133064139/Put/seqid=0 2024-11-09T06:17:44,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742358_1534 (size=12301) 2024-11-09T06:17:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T06:17:44,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T06:17:44,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:44,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:44,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:44,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:44,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:44,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133124662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133124683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133124683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,770 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/43c4330fdb6c43638b8bc8de9ea99809 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/43c4330fdb6c43638b8bc8de9ea99809 2024-11-09T06:17:44,780 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f15639486d79cf453f8c760f89863e07/C of f15639486d79cf453f8c760f89863e07 into 43c4330fdb6c43638b8bc8de9ea99809(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:44,780 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:44,780 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/C, priority=11, startTime=1731133063847; duration=0sec 2024-11-09T06:17:44,780 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:44,780 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:C 2024-11-09T06:17:44,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T06:17:44,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:44,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:44,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:44,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:44,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:44,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T06:17:44,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133124863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133124864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T06:17:44,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:44,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:44,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:44,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:44,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:44,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133124984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:44,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/662d17589a5148359c5e156a2c0ff1cb 2024-11-09T06:17:45,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/3ef9bf5c36bb470d94eb5415d9a508b8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3ef9bf5c36bb470d94eb5415d9a508b8 2024-11-09T06:17:45,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:45,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133125002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:45,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133125003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:45,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3ef9bf5c36bb470d94eb5415d9a508b8, entries=250, sequenceid=389, filesize=16.8 K 2024-11-09T06:17:45,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/236fbac84edf45f189a4c226258cf3f8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/236fbac84edf45f189a4c226258cf3f8 2024-11-09T06:17:45,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/236fbac84edf45f189a4c226258cf3f8, entries=150, sequenceid=389, filesize=12.0 K 2024-11-09T06:17:45,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/662d17589a5148359c5e156a2c0ff1cb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/662d17589a5148359c5e156a2c0ff1cb 2024-11-09T06:17:45,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/662d17589a5148359c5e156a2c0ff1cb, entries=150, sequenceid=389, filesize=12.0 K 2024-11-09T06:17:45,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.84 KB/127836 for f15639486d79cf453f8c760f89863e07 in 883ms, sequenceid=389, compaction requested=true 2024-11-09T06:17:45,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:45,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:45,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:45,022 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:45,023 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:45,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:45,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:45,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:45,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:45,025 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:45,025 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/B is initiating minor compaction (all files) 2024-11-09T06:17:45,025 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:45,025 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/A is initiating minor compaction (all files) 2024-11-09T06:17:45,025 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/A in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:45,026 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b04ad9b1043f4550863ceab041135c00, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/88cb4f03f1f4421294abc17d539f34a0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3ef9bf5c36bb470d94eb5415d9a508b8] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=41.5 K 2024-11-09T06:17:45,026 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/B in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:45,026 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a0e348b9a70745689e2d8e5d5713ae9e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ad8af4ddc5c3439f96df36ed44328165, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/236fbac84edf45f189a4c226258cf3f8] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=36.7 K 2024-11-09T06:17:45,026 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04ad9b1043f4550863ceab041135c00, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731133063307 2024-11-09T06:17:45,026 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a0e348b9a70745689e2d8e5d5713ae9e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731133063307 2024-11-09T06:17:45,027 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88cb4f03f1f4421294abc17d539f34a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731133063440 2024-11-09T06:17:45,027 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting ad8af4ddc5c3439f96df36ed44328165, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731133063440 2024-11-09T06:17:45,027 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ef9bf5c36bb470d94eb5415d9a508b8, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731133064126 2024-11-09T06:17:45,028 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 236fbac84edf45f189a4c226258cf3f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731133064127 2024-11-09T06:17:45,038 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#B#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:45,038 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/a52cddd7bb3a4ff89f0a259258085933 is 50, key is test_row_0/B:col10/1731133064139/Put/seqid=0 2024-11-09T06:17:45,041 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#A#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:45,042 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/27b6792b330b4968aa212ac980dd7745 is 50, key is test_row_0/A:col10/1731133064139/Put/seqid=0 2024-11-09T06:17:45,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742359_1535 (size=13119) 2024-11-09T06:17:45,077 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/a52cddd7bb3a4ff89f0a259258085933 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a52cddd7bb3a4ff89f0a259258085933 2024-11-09T06:17:45,082 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/B of f15639486d79cf453f8c760f89863e07 into a52cddd7bb3a4ff89f0a259258085933(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:45,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742360_1536 (size=13119) 2024-11-09T06:17:45,082 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:45,082 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/B, priority=13, startTime=1731133065023; duration=0sec 2024-11-09T06:17:45,082 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:45,082 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:B 2024-11-09T06:17:45,082 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T06:17:45,083 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T06:17:45,083 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T06:17:45,083 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. because compaction request was cancelled 2024-11-09T06:17:45,083 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:C 2024-11-09T06:17:45,086 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/27b6792b330b4968aa212ac980dd7745 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/27b6792b330b4968aa212ac980dd7745 2024-11-09T06:17:45,092 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/A of f15639486d79cf453f8c760f89863e07 into 27b6792b330b4968aa212ac980dd7745(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:45,092 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:45,092 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/A, priority=13, startTime=1731133065022; duration=0sec 2024-11-09T06:17:45,092 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:45,092 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:A 2024-11-09T06:17:45,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T06:17:45,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:45,111 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=124.84 KB heapSize=354.38 KB 2024-11-09T06:17:45,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:45,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:45,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:45,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:45,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:45,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:45,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/5a04b13b81cb4b73b99d540c10b79b05 is 50, key is test_row_0/A:col10/1731133064332/Put/seqid=0 2024-11-09T06:17:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T06:17:45,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742361_1537 (size=12301) 2024-11-09T06:17:45,142 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/5a04b13b81cb4b73b99d540c10b79b05 2024-11-09T06:17:45,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/d0ba8075a67744259e14b7b020e3a406 is 50, key is test_row_0/B:col10/1731133064332/Put/seqid=0 2024-11-09T06:17:45,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742362_1538 (size=12301) 2024-11-09T06:17:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:45,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:45,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133125411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:45,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133125423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:45,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133125493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:45,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133125512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133125512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:45,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:45,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133125532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:45,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133125542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:45,558 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/d0ba8075a67744259e14b7b020e3a406 2024-11-09T06:17:45,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/16e6afe20a7f4b649ad645ca2983130c is 50, key is test_row_0/C:col10/1731133064332/Put/seqid=0 2024-11-09T06:17:45,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742363_1539 (size=12301) 2024-11-09T06:17:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T06:17:45,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133125752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:45,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133125763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:45,986 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/16e6afe20a7f4b649ad645ca2983130c 2024-11-09T06:17:45,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/5a04b13b81cb4b73b99d540c10b79b05 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5a04b13b81cb4b73b99d540c10b79b05 2024-11-09T06:17:45,993 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5a04b13b81cb4b73b99d540c10b79b05, entries=150, sequenceid=429, filesize=12.0 K 2024-11-09T06:17:45,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/d0ba8075a67744259e14b7b020e3a406 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d0ba8075a67744259e14b7b020e3a406 2024-11-09T06:17:45,998 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d0ba8075a67744259e14b7b020e3a406, entries=150, sequenceid=429, filesize=12.0 K 2024-11-09T06:17:45,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/16e6afe20a7f4b649ad645ca2983130c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/16e6afe20a7f4b649ad645ca2983130c 2024-11-09T06:17:46,004 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/16e6afe20a7f4b649ad645ca2983130c, entries=150, sequenceid=429, filesize=12.0 K 2024-11-09T06:17:46,005 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~124.84 KB/127836, heapSize ~354.34 KB/362840, currentSize=53.67 KB/54960 for f15639486d79cf453f8c760f89863e07 in 893ms, sequenceid=429, compaction requested=true 2024-11-09T06:17:46,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:46,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:46,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-09T06:17:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-09T06:17:46,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-09T06:17:46,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5130 sec 2024-11-09T06:17:46,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5190 sec 2024-11-09T06:17:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:46,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T06:17:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:46,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/f109b288b9cc44d9b6338f8536d52ec2 is 50, key is test_row_0/A:col10/1731133065421/Put/seqid=0 2024-11-09T06:17:46,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742364_1540 (size=14741) 2024-11-09T06:17:46,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/f109b288b9cc44d9b6338f8536d52ec2 2024-11-09T06:17:46,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/2876275f51324281820d9b012f354a56 is 50, key is test_row_0/B:col10/1731133065421/Put/seqid=0 2024-11-09T06:17:46,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742365_1541 (size=12301) 2024-11-09T06:17:46,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/2876275f51324281820d9b012f354a56 2024-11-09T06:17:46,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/8d6e46c51a5343c7a7c8e11069a4c8f5 is 50, key is test_row_0/C:col10/1731133065421/Put/seqid=0 2024-11-09T06:17:46,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742366_1542 (size=12301) 2024-11-09T06:17:46,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/8d6e46c51a5343c7a7c8e11069a4c8f5 2024-11-09T06:17:46,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/f109b288b9cc44d9b6338f8536d52ec2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/f109b288b9cc44d9b6338f8536d52ec2 2024-11-09T06:17:46,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/f109b288b9cc44d9b6338f8536d52ec2, entries=200, sequenceid=441, filesize=14.4 K 2024-11-09T06:17:46,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/2876275f51324281820d9b012f354a56 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/2876275f51324281820d9b012f354a56 2024-11-09T06:17:46,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/2876275f51324281820d9b012f354a56, entries=150, sequenceid=441, filesize=12.0 K 2024-11-09T06:17:46,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/8d6e46c51a5343c7a7c8e11069a4c8f5 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/8d6e46c51a5343c7a7c8e11069a4c8f5 2024-11-09T06:17:46,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/8d6e46c51a5343c7a7c8e11069a4c8f5, entries=150, sequenceid=441, filesize=12.0 K 2024-11-09T06:17:46,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=107.34 KB/109920 for f15639486d79cf453f8c760f89863e07 in 139ms, sequenceid=441, compaction requested=true 2024-11-09T06:17:46,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:46,219 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:46,219 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f15639486d79cf453f8c760f89863e07:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:46,220 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:46,220 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/A is initiating minor compaction (all files) 2024-11-09T06:17:46,220 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:46,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:17:46,220 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/B is initiating minor compaction (all files) 2024-11-09T06:17:46,220 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/B in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:46,221 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a52cddd7bb3a4ff89f0a259258085933, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d0ba8075a67744259e14b7b020e3a406, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/2876275f51324281820d9b012f354a56] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=36.8 K 2024-11-09T06:17:46,221 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/A in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:46,221 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/27b6792b330b4968aa212ac980dd7745, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5a04b13b81cb4b73b99d540c10b79b05, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/f109b288b9cc44d9b6338f8536d52ec2] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=39.2 K 2024-11-09T06:17:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:46,221 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a52cddd7bb3a4ff89f0a259258085933, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731133064127 2024-11-09T06:17:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:46,221 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27b6792b330b4968aa212ac980dd7745, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731133064127 2024-11-09T06:17:46,221 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting d0ba8075a67744259e14b7b020e3a406, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731133064292 2024-11-09T06:17:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:46,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:46,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:46,222 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a04b13b81cb4b73b99d540c10b79b05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731133064292 2024-11-09T06:17:46,222 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2876275f51324281820d9b012f354a56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731133065409 2024-11-09T06:17:46,222 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f109b288b9cc44d9b6338f8536d52ec2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731133065395 2024-11-09T06:17:46,235 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#A#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:46,235 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/67b6562a1d324d6c8b40965e79ea4a24 is 50, key is test_row_0/A:col10/1731133065421/Put/seqid=0 2024-11-09T06:17:46,236 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#B#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:46,236 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/cabc5fc9bb5e4409a3ea1580538e5df1 is 50, key is test_row_0/B:col10/1731133065421/Put/seqid=0 2024-11-09T06:17:46,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/5c99c24572fe41dea223d4a6d17e463b is 50, key is test_row_0/A:col10/1731133066198/Put/seqid=0 2024-11-09T06:17:46,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742367_1543 (size=13221) 2024-11-09T06:17:46,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133126279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:46,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:46,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742368_1544 (size=13221) 2024-11-09T06:17:46,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133126280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:46,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742369_1545 (size=17181) 2024-11-09T06:17:46,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133126393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:46,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133126403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:46,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731133126513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:46,516 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_1' in f15639486d79cf453f8c760f89863e07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:46,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731133126533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:46,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58740 deadline: 1731133126533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:46,546 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_0' in f15639486d79cf453f8c760f89863e07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:46,547 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in f15639486d79cf453f8c760f89863e07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:17:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:46,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133126603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:46,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133126622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T06:17:46,643 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:46,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-09T06:17:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T06:17:46,646 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:46,646 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:46,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:46,673 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/67b6562a1d324d6c8b40965e79ea4a24 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/67b6562a1d324d6c8b40965e79ea4a24 2024-11-09T06:17:46,677 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/A of f15639486d79cf453f8c760f89863e07 into 67b6562a1d324d6c8b40965e79ea4a24(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:46,678 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:46,678 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/A, priority=13, startTime=1731133066218; duration=0sec 2024-11-09T06:17:46,678 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:46,678 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:A 2024-11-09T06:17:46,678 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:17:46,679 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:17:46,679 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): f15639486d79cf453f8c760f89863e07/C is initiating minor compaction (all files) 2024-11-09T06:17:46,679 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f15639486d79cf453f8c760f89863e07/C in TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:46,679 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/43c4330fdb6c43638b8bc8de9ea99809, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/662d17589a5148359c5e156a2c0ff1cb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/16e6afe20a7f4b649ad645ca2983130c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/8d6e46c51a5343c7a7c8e11069a4c8f5] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp, totalSize=48.8 K 2024-11-09T06:17:46,679 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43c4330fdb6c43638b8bc8de9ea99809, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731133063440 2024-11-09T06:17:46,679 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 662d17589a5148359c5e156a2c0ff1cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731133064127 2024-11-09T06:17:46,680 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16e6afe20a7f4b649ad645ca2983130c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731133064292 2024-11-09T06:17:46,680 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d6e46c51a5343c7a7c8e11069a4c8f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731133065409 2024-11-09T06:17:46,688 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/cabc5fc9bb5e4409a3ea1580538e5df1 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/cabc5fc9bb5e4409a3ea1580538e5df1 2024-11-09T06:17:46,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/5c99c24572fe41dea223d4a6d17e463b 2024-11-09T06:17:46,697 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f15639486d79cf453f8c760f89863e07/B of f15639486d79cf453f8c760f89863e07 into cabc5fc9bb5e4409a3ea1580538e5df1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:46,697 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:46,697 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/B, priority=13, startTime=1731133066219; duration=0sec 2024-11-09T06:17:46,698 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:46,698 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:B 2024-11-09T06:17:46,703 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f15639486d79cf453f8c760f89863e07#C#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:46,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/38a285ade0624feda0b39dbfa8132627 is 50, key is test_row_0/B:col10/1731133066198/Put/seqid=0 2024-11-09T06:17:46,704 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/73fae891f8bf477c8e97657513fae004 is 50, key is test_row_0/C:col10/1731133065421/Put/seqid=0 2024-11-09T06:17:46,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742371_1547 (size=13187) 2024-11-09T06:17:46,733 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/73fae891f8bf477c8e97657513fae004 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/73fae891f8bf477c8e97657513fae004 2024-11-09T06:17:46,740 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f15639486d79cf453f8c760f89863e07/C of f15639486d79cf453f8c760f89863e07 into 73fae891f8bf477c8e97657513fae004(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:46,740 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:46,740 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., storeName=f15639486d79cf453f8c760f89863e07/C, priority=12, startTime=1731133066219; duration=0sec 2024-11-09T06:17:46,740 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:46,740 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f15639486d79cf453f8c760f89863e07:C 2024-11-09T06:17:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T06:17:46,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742370_1546 (size=12301) 2024-11-09T06:17:46,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T06:17:46,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:46,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:46,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:46,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:46,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:46,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133126922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:46,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133126943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:46,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T06:17:46,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:46,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:46,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:46,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:46,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T06:17:47,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T06:17:47,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:47,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:47,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:47,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/38a285ade0624feda0b39dbfa8132627 2024-11-09T06:17:47,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/4e60fdce8d90406ea1477572502c25e1 is 50, key is test_row_0/C:col10/1731133066198/Put/seqid=0 2024-11-09T06:17:47,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742372_1548 (size=12301) 2024-11-09T06:17:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T06:17:47,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T06:17:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:47,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T06:17:47,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:47,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:47,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:47,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731133127443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:47,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58768 deadline: 1731133127452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:47,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:47,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:47,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., hostname=4a84bcce7bad,46317,1731132929173, seqNum=2 because the exception is null or not the one we care about 2024-11-09T06:17:47,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T06:17:47,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:47,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. as already flushing 2024-11-09T06:17:47,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:47,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:47,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/4e60fdce8d90406ea1477572502c25e1 2024-11-09T06:17:47,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/5c99c24572fe41dea223d4a6d17e463b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5c99c24572fe41dea223d4a6d17e463b 2024-11-09T06:17:47,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5c99c24572fe41dea223d4a6d17e463b, entries=250, sequenceid=461, filesize=16.8 K 2024-11-09T06:17:47,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/38a285ade0624feda0b39dbfa8132627 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/38a285ade0624feda0b39dbfa8132627 2024-11-09T06:17:47,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/38a285ade0624feda0b39dbfa8132627, entries=150, sequenceid=461, filesize=12.0 K 2024-11-09T06:17:47,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/4e60fdce8d90406ea1477572502c25e1 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/4e60fdce8d90406ea1477572502c25e1 2024-11-09T06:17:47,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/4e60fdce8d90406ea1477572502c25e1, entries=150, sequenceid=461, filesize=12.0 K 2024-11-09T06:17:47,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f15639486d79cf453f8c760f89863e07 in 1417ms, sequenceid=461, compaction requested=false 2024-11-09T06:17:47,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:47,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T06:17:47,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:47,742 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T06:17:47,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:47,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:47,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:47,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:47,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:47,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:47,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d850a762a4604cb1bad443c99d21e86e is 50, key is test_row_0/A:col10/1731133066261/Put/seqid=0 2024-11-09T06:17:47,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742373_1549 (size=12301) 2024-11-09T06:17:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T06:17:48,164 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d850a762a4604cb1bad443c99d21e86e 2024-11-09T06:17:48,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/6c878e69e5454540b83a3c18744275dc is 50, key is test_row_0/B:col10/1731133066261/Put/seqid=0 2024-11-09T06:17:48,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742374_1550 (size=12301) 2024-11-09T06:17:48,184 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/6c878e69e5454540b83a3c18744275dc 2024-11-09T06:17:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/bcb583f2e2bb4a6b9777afc6e6a9fceb is 50, key is test_row_0/C:col10/1731133066261/Put/seqid=0 2024-11-09T06:17:48,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742375_1551 (size=12301) 2024-11-09T06:17:48,228 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/bcb583f2e2bb4a6b9777afc6e6a9fceb 2024-11-09T06:17:48,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d850a762a4604cb1bad443c99d21e86e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d850a762a4604cb1bad443c99d21e86e 2024-11-09T06:17:48,242 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d850a762a4604cb1bad443c99d21e86e, entries=150, sequenceid=476, filesize=12.0 K 2024-11-09T06:17:48,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/6c878e69e5454540b83a3c18744275dc as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/6c878e69e5454540b83a3c18744275dc 2024-11-09T06:17:48,247 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/6c878e69e5454540b83a3c18744275dc, entries=150, sequenceid=476, filesize=12.0 K 2024-11-09T06:17:48,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/bcb583f2e2bb4a6b9777afc6e6a9fceb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/bcb583f2e2bb4a6b9777afc6e6a9fceb 2024-11-09T06:17:48,256 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/bcb583f2e2bb4a6b9777afc6e6a9fceb, entries=150, sequenceid=476, filesize=12.0 K 2024-11-09T06:17:48,257 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for f15639486d79cf453f8c760f89863e07 in 514ms, sequenceid=476, compaction requested=true 2024-11-09T06:17:48,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for f15639486d79cf453f8c760f89863e07: 2024-11-09T06:17:48,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:48,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-09T06:17:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-09T06:17:48,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-09T06:17:48,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-09T06:17:48,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.6190 sec 2024-11-09T06:17:48,292 INFO [Thread-2041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2041. 2024-11-09T06:17:48,292 DEBUG [Thread-2041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:48,292 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,292 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:48,293 INFO [Thread-2045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2045. 2024-11-09T06:17:48,293 DEBUG [Thread-2045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:48,293 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:48,293 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,293 INFO [Thread-2043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2043. 2024-11-09T06:17:48,293 DEBUG [Thread-2043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:48,293 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,293 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:48,295 INFO [Thread-2039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2039. 2024-11-09T06:17:48,295 DEBUG [Thread-2039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:48,296 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,296 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:48,296 INFO [Thread-2047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2047. 2024-11-09T06:17:48,296 DEBUG [Thread-2047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:48,296 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,296 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:48,463 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-11-09T06:17:48,464 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:48,464 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,464 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:48,474 INFO [Thread-2036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2036. 2024-11-09T06:17:48,474 DEBUG [Thread-2036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:48,474 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,474 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:48,533 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-11-09T06:17:48,533 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:48,533 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,533 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,533 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:48,564 INFO [Thread-2034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2034. 2024-11-09T06:17:48,564 DEBUG [Thread-2034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:48,564 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,564 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,564 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:48,574 INFO [Thread-2028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2028. 2024-11-09T06:17:48,574 DEBUG [Thread-2028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:17:48,574 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,574 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T06:17:48,793 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 168 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 930 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2790 rows 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 923 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2769 rows 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 926 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2778 rows 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 927 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2781 rows 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 929 2024-11-09T06:17:48,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2787 rows 2024-11-09T06:17:48,793 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T06:17:48,793 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T06:17:48,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:48,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:17:48,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T06:17:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T06:17:48,797 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133068796"}]},"ts":"1731133068796"} 2024-11-09T06:17:48,798 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T06:17:48,798 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T06:17:48,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T06:17:48,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f15639486d79cf453f8c760f89863e07, UNASSIGN}] 2024-11-09T06:17:48,800 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f15639486d79cf453f8c760f89863e07, UNASSIGN 2024-11-09T06:17:48,801 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=f15639486d79cf453f8c760f89863e07, regionState=CLOSING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:48,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f15639486d79cf453f8c760f89863e07, UNASSIGN because future has completed 2024-11-09T06:17:48,802 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T06:17:48,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:17:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T06:17:48,954 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:48,954 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T06:17:48,954 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing f15639486d79cf453f8c760f89863e07, disabling compactions & flushes 2024-11-09T06:17:48,954 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:48,954 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:48,954 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. after waiting 0 ms 2024-11-09T06:17:48,954 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:48,954 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing f15639486d79cf453f8c760f89863e07 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T06:17:48,954 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=A 2024-11-09T06:17:48,954 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:48,954 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=B 2024-11-09T06:17:48,954 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:48,955 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f15639486d79cf453f8c760f89863e07, store=C 2024-11-09T06:17:48,955 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:48,958 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d9a67e2e21414d1ea3cf35b4dadf7ac2 is 50, key is test_row_0/A:col10/1731133068563/Put/seqid=0 2024-11-09T06:17:48,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742376_1552 (size=12301) 2024-11-09T06:17:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T06:17:49,361 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d9a67e2e21414d1ea3cf35b4dadf7ac2 2024-11-09T06:17:49,367 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/63b3d55b5a6d487fa42777080b967ee2 is 50, key is test_row_0/B:col10/1731133068563/Put/seqid=0 2024-11-09T06:17:49,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742377_1553 (size=12301) 2024-11-09T06:17:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T06:17:49,770 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/63b3d55b5a6d487fa42777080b967ee2 2024-11-09T06:17:49,776 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/37ef215219d049dfa1fb60fca2ae31b1 is 50, key is test_row_0/C:col10/1731133068563/Put/seqid=0 2024-11-09T06:17:49,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742378_1554 (size=12301) 2024-11-09T06:17:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T06:17:50,182 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/37ef215219d049dfa1fb60fca2ae31b1 2024-11-09T06:17:50,186 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/A/d9a67e2e21414d1ea3cf35b4dadf7ac2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d9a67e2e21414d1ea3cf35b4dadf7ac2 2024-11-09T06:17:50,188 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d9a67e2e21414d1ea3cf35b4dadf7ac2, entries=150, sequenceid=484, filesize=12.0 K 2024-11-09T06:17:50,189 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/B/63b3d55b5a6d487fa42777080b967ee2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/63b3d55b5a6d487fa42777080b967ee2 2024-11-09T06:17:50,191 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/63b3d55b5a6d487fa42777080b967ee2, entries=150, sequenceid=484, filesize=12.0 K 2024-11-09T06:17:50,192 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/.tmp/C/37ef215219d049dfa1fb60fca2ae31b1 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/37ef215219d049dfa1fb60fca2ae31b1 2024-11-09T06:17:50,196 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/37ef215219d049dfa1fb60fca2ae31b1, entries=150, sequenceid=484, filesize=12.0 K 2024-11-09T06:17:50,197 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f15639486d79cf453f8c760f89863e07 in 1243ms, sequenceid=484, compaction requested=true 2024-11-09T06:17:50,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/044e2222055c4703a2a216c7e6a076ec, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/555ee8121df2464cac3437c89ffe5e5e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/316531d471da411d96259a287f38359d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/2be67310c49f499ea2ab7f4a2eb5ea84, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8abf3c2bc4c2476895a78edd66e5b011, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b82b2f1fdf6741439a150b64609abb77, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3e700e6b26154cc7b1bb1307b956b6a2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c0194c091f83421d8f653250942a7957, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/86daaf2e91704bdf84d1a07b90398711, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d07d94ac22994030922904102ae0731d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/6b9aaad708704b61910da9009a33034a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/e6f27f9acca448299507f690cdebe87d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/84c94d35389d448592291d756cc59b1e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b5ade32801c2417297ea3445c0024d13, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/dfe8fdee7bd64d409c096933c646aeee, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1ad048a474fa40688c7e54e15444d85d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c3bc4a9bab384e6b8790741206666b4a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/6e04e39d639c4e4d9b5668473b2a54bf, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/46e8cfd2f1434495971df4217394af03, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d798d6425b7f4915a958e9aac67b16eb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1a80cdfa494f4bdca9fe1cb48525a849, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b04ad9b1043f4550863ceab041135c00, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/88cb4f03f1f4421294abc17d539f34a0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3ef9bf5c36bb470d94eb5415d9a508b8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/27b6792b330b4968aa212ac980dd7745, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5a04b13b81cb4b73b99d540c10b79b05, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/f109b288b9cc44d9b6338f8536d52ec2] to archive 2024-11-09T06:17:50,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:17:50,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/044e2222055c4703a2a216c7e6a076ec to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/044e2222055c4703a2a216c7e6a076ec 2024-11-09T06:17:50,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/555ee8121df2464cac3437c89ffe5e5e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/555ee8121df2464cac3437c89ffe5e5e 2024-11-09T06:17:50,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/316531d471da411d96259a287f38359d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/316531d471da411d96259a287f38359d 2024-11-09T06:17:50,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/2be67310c49f499ea2ab7f4a2eb5ea84 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/2be67310c49f499ea2ab7f4a2eb5ea84 2024-11-09T06:17:50,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8abf3c2bc4c2476895a78edd66e5b011 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/8abf3c2bc4c2476895a78edd66e5b011 2024-11-09T06:17:50,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b82b2f1fdf6741439a150b64609abb77 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b82b2f1fdf6741439a150b64609abb77 2024-11-09T06:17:50,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3e700e6b26154cc7b1bb1307b956b6a2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3e700e6b26154cc7b1bb1307b956b6a2 2024-11-09T06:17:50,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c0194c091f83421d8f653250942a7957 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c0194c091f83421d8f653250942a7957 2024-11-09T06:17:50,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/86daaf2e91704bdf84d1a07b90398711 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/86daaf2e91704bdf84d1a07b90398711 2024-11-09T06:17:50,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d07d94ac22994030922904102ae0731d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d07d94ac22994030922904102ae0731d 2024-11-09T06:17:50,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/6b9aaad708704b61910da9009a33034a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/6b9aaad708704b61910da9009a33034a 2024-11-09T06:17:50,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/e6f27f9acca448299507f690cdebe87d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/e6f27f9acca448299507f690cdebe87d 2024-11-09T06:17:50,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/84c94d35389d448592291d756cc59b1e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/84c94d35389d448592291d756cc59b1e 2024-11-09T06:17:50,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b5ade32801c2417297ea3445c0024d13 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b5ade32801c2417297ea3445c0024d13 2024-11-09T06:17:50,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/dfe8fdee7bd64d409c096933c646aeee to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/dfe8fdee7bd64d409c096933c646aeee 2024-11-09T06:17:50,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1ad048a474fa40688c7e54e15444d85d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1ad048a474fa40688c7e54e15444d85d 2024-11-09T06:17:50,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c3bc4a9bab384e6b8790741206666b4a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/c3bc4a9bab384e6b8790741206666b4a 2024-11-09T06:17:50,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/6e04e39d639c4e4d9b5668473b2a54bf to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/6e04e39d639c4e4d9b5668473b2a54bf 2024-11-09T06:17:50,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/46e8cfd2f1434495971df4217394af03 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/46e8cfd2f1434495971df4217394af03 2024-11-09T06:17:50,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d798d6425b7f4915a958e9aac67b16eb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d798d6425b7f4915a958e9aac67b16eb 2024-11-09T06:17:50,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1a80cdfa494f4bdca9fe1cb48525a849 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/1a80cdfa494f4bdca9fe1cb48525a849 2024-11-09T06:17:50,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b04ad9b1043f4550863ceab041135c00 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/b04ad9b1043f4550863ceab041135c00 2024-11-09T06:17:50,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/88cb4f03f1f4421294abc17d539f34a0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/88cb4f03f1f4421294abc17d539f34a0 2024-11-09T06:17:50,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3ef9bf5c36bb470d94eb5415d9a508b8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/3ef9bf5c36bb470d94eb5415d9a508b8 2024-11-09T06:17:50,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/27b6792b330b4968aa212ac980dd7745 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/27b6792b330b4968aa212ac980dd7745 2024-11-09T06:17:50,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5a04b13b81cb4b73b99d540c10b79b05 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5a04b13b81cb4b73b99d540c10b79b05 2024-11-09T06:17:50,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/f109b288b9cc44d9b6338f8536d52ec2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/f109b288b9cc44d9b6338f8536d52ec2 2024-11-09T06:17:50,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/2a3f4b7de6364c5594a63a73ca4d5874, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/05b6c7ef536044f39cfeccdf5c4015a2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a65002775b8d48cc85ea18c5ae911f65, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/28a60559a0f54d2c9b24be225c3efe8d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f16e4d2d6b464ca4b6fc0a059fca92f3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/60845caf53f74d279a9a6442e6c4bb87, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/0d18f3832c0c4404add338c26f9ad522, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f24850ccdbd24a5fb4d7995770a9aa4b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/cd973cb3b6854c65be18e5184bceef8d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/473a04af862943b0a4a5aef41e0f86b7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3a6d85250d254373a533246c03c84327, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f4d37c3ee6094b87aed326fa5d289356, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f6e2a164eec04bda894949ff557e28b0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/033cd170d07145c09871681d0eb4d894, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d8607dbcbb06405e9c2e75cc36e0d50b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/4d573700d26e479e8605377d1431d96d, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/82b9b0cb201a408eaeca215545870b4a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/93dac15778244c438ca8c57c0abe6679, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3c966174be9f43fcbe64a3cd12279374, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d18d300109ab4778a8914ede5db2f0fa, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a0e348b9a70745689e2d8e5d5713ae9e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/9e726fcf655547e691e7a222ef072b99, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ad8af4ddc5c3439f96df36ed44328165, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a52cddd7bb3a4ff89f0a259258085933, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/236fbac84edf45f189a4c226258cf3f8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d0ba8075a67744259e14b7b020e3a406, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/2876275f51324281820d9b012f354a56] to archive 2024-11-09T06:17:50,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:17:50,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/2a3f4b7de6364c5594a63a73ca4d5874 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/2a3f4b7de6364c5594a63a73ca4d5874 2024-11-09T06:17:50,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/05b6c7ef536044f39cfeccdf5c4015a2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/05b6c7ef536044f39cfeccdf5c4015a2 2024-11-09T06:17:50,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a65002775b8d48cc85ea18c5ae911f65 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a65002775b8d48cc85ea18c5ae911f65 2024-11-09T06:17:50,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/28a60559a0f54d2c9b24be225c3efe8d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/28a60559a0f54d2c9b24be225c3efe8d 2024-11-09T06:17:50,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f16e4d2d6b464ca4b6fc0a059fca92f3 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f16e4d2d6b464ca4b6fc0a059fca92f3 2024-11-09T06:17:50,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/60845caf53f74d279a9a6442e6c4bb87 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/60845caf53f74d279a9a6442e6c4bb87 2024-11-09T06:17:50,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/0d18f3832c0c4404add338c26f9ad522 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/0d18f3832c0c4404add338c26f9ad522 2024-11-09T06:17:50,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f24850ccdbd24a5fb4d7995770a9aa4b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f24850ccdbd24a5fb4d7995770a9aa4b 2024-11-09T06:17:50,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/cd973cb3b6854c65be18e5184bceef8d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/cd973cb3b6854c65be18e5184bceef8d 2024-11-09T06:17:50,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/473a04af862943b0a4a5aef41e0f86b7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/473a04af862943b0a4a5aef41e0f86b7 2024-11-09T06:17:50,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3a6d85250d254373a533246c03c84327 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3a6d85250d254373a533246c03c84327 2024-11-09T06:17:50,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f4d37c3ee6094b87aed326fa5d289356 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f4d37c3ee6094b87aed326fa5d289356 2024-11-09T06:17:50,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f6e2a164eec04bda894949ff557e28b0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/f6e2a164eec04bda894949ff557e28b0 2024-11-09T06:17:50,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/033cd170d07145c09871681d0eb4d894 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/033cd170d07145c09871681d0eb4d894 2024-11-09T06:17:50,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d8607dbcbb06405e9c2e75cc36e0d50b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d8607dbcbb06405e9c2e75cc36e0d50b 2024-11-09T06:17:50,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/4d573700d26e479e8605377d1431d96d to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/4d573700d26e479e8605377d1431d96d 2024-11-09T06:17:50,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/82b9b0cb201a408eaeca215545870b4a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/82b9b0cb201a408eaeca215545870b4a 2024-11-09T06:17:50,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/93dac15778244c438ca8c57c0abe6679 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/93dac15778244c438ca8c57c0abe6679 2024-11-09T06:17:50,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3c966174be9f43fcbe64a3cd12279374 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/3c966174be9f43fcbe64a3cd12279374 2024-11-09T06:17:50,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d18d300109ab4778a8914ede5db2f0fa to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d18d300109ab4778a8914ede5db2f0fa 2024-11-09T06:17:50,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a0e348b9a70745689e2d8e5d5713ae9e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a0e348b9a70745689e2d8e5d5713ae9e 2024-11-09T06:17:50,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/9e726fcf655547e691e7a222ef072b99 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/9e726fcf655547e691e7a222ef072b99 2024-11-09T06:17:50,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ad8af4ddc5c3439f96df36ed44328165 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/ad8af4ddc5c3439f96df36ed44328165 2024-11-09T06:17:50,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a52cddd7bb3a4ff89f0a259258085933 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/a52cddd7bb3a4ff89f0a259258085933 2024-11-09T06:17:50,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/236fbac84edf45f189a4c226258cf3f8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/236fbac84edf45f189a4c226258cf3f8 2024-11-09T06:17:50,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d0ba8075a67744259e14b7b020e3a406 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/d0ba8075a67744259e14b7b020e3a406 2024-11-09T06:17:50,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/2876275f51324281820d9b012f354a56 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/2876275f51324281820d9b012f354a56 2024-11-09T06:17:50,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/792ac5c4a00448fca2c0bd35c16b3c16, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/ab6fc06aeef44b91a76165c850ae11d4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/1aac2638a2574aeea247de0a1c9562e7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7d8651ef67b240198ad7d35c7f36164b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/05a92dbd60c04650acea7184b4e7bead, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/443beb14cd9c4a54a6c588011e5be92a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d05ee21627eb413d8fbab153be9aee8c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/1f46d07274f240a29b504b482a3e859b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d2b946380e574a44a1a52ba4439052fc, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/078551e56a4d478daa00f736d3395397, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9b46174592f3438b8059b956368c0bb5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/25de12a932c944179ece27131fed1d96, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/033b069606ab4768ac2a202e6586a22f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9e16e2779a9445fc8112da7bd6755649, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/c7292ceb7b904cb1969e71f34a1748da, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7a5980968061484ba93a03b2d5edeaed, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/c39859eaf90f47ebad729656a11b5ab2, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/18ac09105c25440582f59f93b8daa0ad, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/79c2d4ae9e02488ebcb3a48d16b18b47, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/eb9e9d3832c14f6190fac47247c1ccfb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/fc04ad06f2014e87877e5b7b01529f0b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/43c4330fdb6c43638b8bc8de9ea99809, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/b0b8aa6e09484821a46d1dcbf826eb33, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/662d17589a5148359c5e156a2c0ff1cb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/16e6afe20a7f4b649ad645ca2983130c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/8d6e46c51a5343c7a7c8e11069a4c8f5] to archive 2024-11-09T06:17:50,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:17:50,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/792ac5c4a00448fca2c0bd35c16b3c16 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/792ac5c4a00448fca2c0bd35c16b3c16 2024-11-09T06:17:50,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/ab6fc06aeef44b91a76165c850ae11d4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/ab6fc06aeef44b91a76165c850ae11d4 2024-11-09T06:17:50,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/1aac2638a2574aeea247de0a1c9562e7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/1aac2638a2574aeea247de0a1c9562e7 2024-11-09T06:17:50,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7d8651ef67b240198ad7d35c7f36164b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7d8651ef67b240198ad7d35c7f36164b 2024-11-09T06:17:50,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/05a92dbd60c04650acea7184b4e7bead to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/05a92dbd60c04650acea7184b4e7bead 2024-11-09T06:17:50,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/443beb14cd9c4a54a6c588011e5be92a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/443beb14cd9c4a54a6c588011e5be92a 2024-11-09T06:17:50,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d05ee21627eb413d8fbab153be9aee8c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d05ee21627eb413d8fbab153be9aee8c 2024-11-09T06:17:50,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/1f46d07274f240a29b504b482a3e859b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/1f46d07274f240a29b504b482a3e859b 2024-11-09T06:17:50,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d2b946380e574a44a1a52ba4439052fc to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/d2b946380e574a44a1a52ba4439052fc 2024-11-09T06:17:50,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/078551e56a4d478daa00f736d3395397 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/078551e56a4d478daa00f736d3395397 2024-11-09T06:17:50,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9b46174592f3438b8059b956368c0bb5 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9b46174592f3438b8059b956368c0bb5 2024-11-09T06:17:50,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/25de12a932c944179ece27131fed1d96 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/25de12a932c944179ece27131fed1d96 2024-11-09T06:17:50,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/033b069606ab4768ac2a202e6586a22f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/033b069606ab4768ac2a202e6586a22f 2024-11-09T06:17:50,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9e16e2779a9445fc8112da7bd6755649 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/9e16e2779a9445fc8112da7bd6755649 2024-11-09T06:17:50,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/c7292ceb7b904cb1969e71f34a1748da to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/c7292ceb7b904cb1969e71f34a1748da 2024-11-09T06:17:50,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7a5980968061484ba93a03b2d5edeaed to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/7a5980968061484ba93a03b2d5edeaed 2024-11-09T06:17:50,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/c39859eaf90f47ebad729656a11b5ab2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/c39859eaf90f47ebad729656a11b5ab2 2024-11-09T06:17:50,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/18ac09105c25440582f59f93b8daa0ad to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/18ac09105c25440582f59f93b8daa0ad 2024-11-09T06:17:50,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/79c2d4ae9e02488ebcb3a48d16b18b47 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/79c2d4ae9e02488ebcb3a48d16b18b47 2024-11-09T06:17:50,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/eb9e9d3832c14f6190fac47247c1ccfb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/eb9e9d3832c14f6190fac47247c1ccfb 2024-11-09T06:17:50,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/fc04ad06f2014e87877e5b7b01529f0b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/fc04ad06f2014e87877e5b7b01529f0b 2024-11-09T06:17:50,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/43c4330fdb6c43638b8bc8de9ea99809 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/43c4330fdb6c43638b8bc8de9ea99809 2024-11-09T06:17:50,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/b0b8aa6e09484821a46d1dcbf826eb33 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/b0b8aa6e09484821a46d1dcbf826eb33 2024-11-09T06:17:50,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/662d17589a5148359c5e156a2c0ff1cb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/662d17589a5148359c5e156a2c0ff1cb 2024-11-09T06:17:50,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/16e6afe20a7f4b649ad645ca2983130c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/16e6afe20a7f4b649ad645ca2983130c 2024-11-09T06:17:50,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/8d6e46c51a5343c7a7c8e11069a4c8f5 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/8d6e46c51a5343c7a7c8e11069a4c8f5 2024-11-09T06:17:50,293 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/recovered.edits/487.seqid, newMaxSeqId=487, maxSeqId=1 2024-11-09T06:17:50,294 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. 2024-11-09T06:17:50,294 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for f15639486d79cf453f8c760f89863e07: Waiting for close lock at 1731133068954Running coprocessor pre-close hooks at 1731133068954Disabling compacts and flushes for region at 1731133068954Disabling writes for close at 1731133068954Obtaining lock to block concurrent updates at 1731133068954Preparing flush snapshotting stores in f15639486d79cf453f8c760f89863e07 at 1731133068954Finished memstore snapshotting TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731133068955 (+1 ms)Flushing stores of TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07. at 1731133068955Flushing f15639486d79cf453f8c760f89863e07/A: creating writer at 1731133068955Flushing f15639486d79cf453f8c760f89863e07/A: appending metadata at 1731133068957 (+2 ms)Flushing f15639486d79cf453f8c760f89863e07/A: closing flushed file at 1731133068957Flushing f15639486d79cf453f8c760f89863e07/B: creating writer at 1731133069364 (+407 ms)Flushing f15639486d79cf453f8c760f89863e07/B: appending metadata at 1731133069366 (+2 ms)Flushing f15639486d79cf453f8c760f89863e07/B: closing flushed file at 1731133069366Flushing f15639486d79cf453f8c760f89863e07/C: creating writer at 1731133069772 (+406 ms)Flushing f15639486d79cf453f8c760f89863e07/C: appending metadata at 1731133069775 (+3 ms)Flushing f15639486d79cf453f8c760f89863e07/C: closing flushed file at 1731133069775Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35a77f46: reopening flushed file at 1731133070185 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f6e588d: reopening flushed file at 1731133070188 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22da53a8: reopening flushed file at 1731133070191 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f15639486d79cf453f8c760f89863e07 in 1243ms, sequenceid=484, compaction requested=true at 1731133070197 (+6 ms)Writing region close event to WAL at 1731133070290 (+93 ms)Running coprocessor post-close hooks at 1731133070294 (+4 ms)Closed at 1731133070294 2024-11-09T06:17:50,296 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:50,296 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=f15639486d79cf453f8c760f89863e07, regionState=CLOSED 2024-11-09T06:17:50,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:17:50,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-09T06:17:50,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure f15639486d79cf453f8c760f89863e07, server=4a84bcce7bad,46317,1731132929173 in 1.4970 sec 2024-11-09T06:17:50,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-09T06:17:50,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f15639486d79cf453f8c760f89863e07, UNASSIGN in 1.5020 sec 2024-11-09T06:17:50,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-09T06:17:50,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5050 sec 2024-11-09T06:17:50,305 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133070305"}]},"ts":"1731133070305"} 2024-11-09T06:17:50,306 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T06:17:50,306 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T06:17:50,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5130 sec 2024-11-09T06:17:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T06:17:50,943 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:50,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T06:17:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:50,945 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-09T06:17:50,945 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:50,947 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:50,949 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/recovered.edits] 2024-11-09T06:17:50,951 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5c99c24572fe41dea223d4a6d17e463b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/5c99c24572fe41dea223d4a6d17e463b 2024-11-09T06:17:50,952 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/67b6562a1d324d6c8b40965e79ea4a24 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/67b6562a1d324d6c8b40965e79ea4a24 2024-11-09T06:17:50,953 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d850a762a4604cb1bad443c99d21e86e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d850a762a4604cb1bad443c99d21e86e 2024-11-09T06:17:50,954 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d9a67e2e21414d1ea3cf35b4dadf7ac2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/A/d9a67e2e21414d1ea3cf35b4dadf7ac2 2024-11-09T06:17:50,956 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/38a285ade0624feda0b39dbfa8132627 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/38a285ade0624feda0b39dbfa8132627 2024-11-09T06:17:50,957 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/63b3d55b5a6d487fa42777080b967ee2 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/63b3d55b5a6d487fa42777080b967ee2 2024-11-09T06:17:50,958 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/6c878e69e5454540b83a3c18744275dc to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/6c878e69e5454540b83a3c18744275dc 2024-11-09T06:17:50,959 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/cabc5fc9bb5e4409a3ea1580538e5df1 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/B/cabc5fc9bb5e4409a3ea1580538e5df1 2024-11-09T06:17:50,960 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/37ef215219d049dfa1fb60fca2ae31b1 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/37ef215219d049dfa1fb60fca2ae31b1 2024-11-09T06:17:50,961 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/4e60fdce8d90406ea1477572502c25e1 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/4e60fdce8d90406ea1477572502c25e1 2024-11-09T06:17:50,962 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/73fae891f8bf477c8e97657513fae004 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/73fae891f8bf477c8e97657513fae004 2024-11-09T06:17:50,963 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/bcb583f2e2bb4a6b9777afc6e6a9fceb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/C/bcb583f2e2bb4a6b9777afc6e6a9fceb 2024-11-09T06:17:50,967 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/recovered.edits/487.seqid to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07/recovered.edits/487.seqid 2024-11-09T06:17:50,968 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/f15639486d79cf453f8c760f89863e07 2024-11-09T06:17:50,968 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T06:17:50,971 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:50,974 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T06:17:50,976 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T06:17:50,977 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:50,977 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T06:17:50,978 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731133070977"}]},"ts":"9223372036854775807"} 2024-11-09T06:17:50,980 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T06:17:50,980 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f15639486d79cf453f8c760f89863e07, NAME => 'TestAcidGuarantees,,1731133047102.f15639486d79cf453f8c760f89863e07.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T06:17:50,980 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T06:17:50,980 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731133070980"}]},"ts":"9223372036854775807"} 2024-11-09T06:17:50,982 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T06:17:50,983 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:50,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-09T06:17:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-09T06:17:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T06:17:51,055 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:51,062 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252 (was 252), OpenFileDescriptor=455 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=751 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=632 (was 944) 2024-11-09T06:17:51,072 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=751, ProcessCount=11, AvailableMemoryMB=632 2024-11-09T06:17:51,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T06:17:51,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T06:17:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:51,075 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T06:17:51,075 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:51,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-09T06:17:51,076 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T06:17:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T06:17:51,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742379_1555 (size=1061) 2024-11-09T06:17:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T06:17:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T06:17:51,484 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f 2024-11-09T06:17:51,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742380_1556 (size=53) 2024-11-09T06:17:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T06:17:51,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:17:51,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, disabling compactions & flushes 2024-11-09T06:17:51,894 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:51,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:51,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. after waiting 0 ms 2024-11-09T06:17:51,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:51,894 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:51,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: Waiting for close lock at 1731133071894Disabling compacts and flushes for region at 1731133071894Disabling writes for close at 1731133071894Writing region close event to WAL at 1731133071894Closed at 1731133071894 2024-11-09T06:17:51,895 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T06:17:51,896 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731133071895"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731133071895"}]},"ts":"1731133071895"} 2024-11-09T06:17:51,897 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T06:17:51,898 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T06:17:51,898 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133071898"}]},"ts":"1731133071898"} 2024-11-09T06:17:51,899 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T06:17:51,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, ASSIGN}] 2024-11-09T06:17:51,900 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, ASSIGN 2024-11-09T06:17:51,901 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, ASSIGN; state=OFFLINE, location=4a84bcce7bad,46317,1731132929173; forceNewPlan=false, retain=false 2024-11-09T06:17:52,051 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, regionState=OPENING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:52,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, ASSIGN because future has completed 2024-11-09T06:17:52,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:17:52,207 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:52,207 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} 2024-11-09T06:17:52,207 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,208 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:17:52,208 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,208 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,209 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,210 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:52,210 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca columnFamilyName A 2024-11-09T06:17:52,210 DEBUG [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:52,210 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(327): Store=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:52,211 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,211 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:52,211 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca columnFamilyName B 2024-11-09T06:17:52,211 DEBUG [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:52,212 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(327): Store=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:52,212 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,212 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:52,213 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca columnFamilyName C 2024-11-09T06:17:52,213 DEBUG [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T06:17:52,213 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(327): Store=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:52,213 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:52,213 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,214 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,214 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,215 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,215 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,215 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T06:17:52,216 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,217 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T06:17:52,218 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 1aeacadfe7a3b9ebc8c8b0e6e39a10ca; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75143453, jitterRate=0.11972470581531525}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T06:17:52,218 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:52,218 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: Running coprocessor pre-open hook at 1731133072208Writing region info on filesystem at 1731133072208Initializing all the Stores at 1731133072208Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133072208Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133072209 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133072209Cleaning up temporary data from old regions at 1731133072215 (+6 ms)Running coprocessor post-open hooks at 1731133072218 (+3 ms)Region opened successfully at 1731133072218 2024-11-09T06:17:52,219 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., pid=138, masterSystemTime=1731133072204 2024-11-09T06:17:52,220 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:52,220 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:52,221 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, regionState=OPEN, openSeqNum=2, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:52,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:17:52,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-09T06:17:52,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 in 169 msec 2024-11-09T06:17:52,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-09T06:17:52,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, ASSIGN in 325 msec 2024-11-09T06:17:52,226 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T06:17:52,226 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133072226"}]},"ts":"1731133072226"} 2024-11-09T06:17:52,227 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T06:17:52,228 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T06:17:52,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-09T06:17:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-09T06:17:53,222 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:53,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:53,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:53,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:53,224 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:53,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:53,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:53,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ab5178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:53,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:53,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:53,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:53,226 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2640471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:53,226 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:53,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:53,229 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T06:17:53,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a84bcce7bad,37069,1731132928314 2024-11-09T06:17:53,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ff4ca9f 2024-11-09T06:17:53,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T06:17:53,230 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T06:17:53,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T06:17:53,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T06:17:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T06:17:53,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742381_1557 (size=1097) 2024-11-09T06:17:53,650 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-09T06:17:53,650 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-09T06:17:53,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T06:17:53,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, REOPEN/MOVE}] 2024-11-09T06:17:53,654 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, REOPEN/MOVE 2024-11-09T06:17:53,655 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, regionState=CLOSING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:53,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, REOPEN/MOVE because future has completed 2024-11-09T06:17:53,656 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T06:17:53,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:17:53,808 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:53,808 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T06:17:53,808 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, disabling compactions & flushes 2024-11-09T06:17:53,808 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:53,808 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:53,808 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. after waiting 0 ms 2024-11-09T06:17:53,808 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:53,817 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T06:17:53,817 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:53,817 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: Waiting for close lock at 1731133073808Running coprocessor pre-close hooks at 1731133073808Disabling compacts and flushes for region at 1731133073808Disabling writes for close at 1731133073808Writing region close event to WAL at 1731133073812 (+4 ms)Running coprocessor post-close hooks at 1731133073817 (+5 ms)Closed at 1731133073817 2024-11-09T06:17:53,817 WARN [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca to self. 2024-11-09T06:17:53,819 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:53,820 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, regionState=CLOSED 2024-11-09T06:17:53,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:17:53,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-09T06:17:53,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 in 166 msec 2024-11-09T06:17:53,823 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, REOPEN/MOVE; state=CLOSED, location=4a84bcce7bad,46317,1731132929173; forceNewPlan=false, retain=true 2024-11-09T06:17:53,974 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, regionState=OPENING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:53,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, REOPEN/MOVE because future has completed 2024-11-09T06:17:53,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:17:54,130 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,130 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} 2024-11-09T06:17:54,130 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,130 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T06:17:54,130 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,130 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,131 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,131 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:54,132 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca columnFamilyName A 2024-11-09T06:17:54,132 DEBUG [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:54,133 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(327): Store=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:54,133 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,133 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:54,133 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca columnFamilyName B 2024-11-09T06:17:54,134 DEBUG [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:54,134 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(327): Store=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:54,134 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,134 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T06:17:54,134 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aeacadfe7a3b9ebc8c8b0e6e39a10ca columnFamilyName C 2024-11-09T06:17:54,134 DEBUG [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:54,135 INFO [StoreOpener-1aeacadfe7a3b9ebc8c8b0e6e39a10ca-1 {}] regionserver.HStore(327): Store=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T06:17:54,135 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,135 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,135 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,136 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,138 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,138 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,138 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T06:17:54,140 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,140 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 1aeacadfe7a3b9ebc8c8b0e6e39a10ca; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58902923, jitterRate=-0.12227804958820343}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T06:17:54,140 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,141 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: Running coprocessor pre-open hook at 1731133074130Writing region info on filesystem at 1731133074130Initializing all the Stores at 1731133074131 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133074131Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133074131Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731133074131Cleaning up temporary data from old regions at 1731133074138 (+7 ms)Running coprocessor post-open hooks at 1731133074140 (+2 ms)Region opened successfully at 1731133074141 (+1 ms) 2024-11-09T06:17:54,142 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., pid=143, masterSystemTime=1731133074127 2024-11-09T06:17:54,143 DEBUG [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,144 INFO [RS_OPEN_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,144 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, regionState=OPEN, openSeqNum=5, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:17:54,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-09T06:17:54,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 in 171 msec 2024-11-09T06:17:54,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-09T06:17:54,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, REOPEN/MOVE in 494 msec 2024-11-09T06:17:54,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-09T06:17:54,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-11-09T06:17:54,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 921 msec 2024-11-09T06:17:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-09T06:17:54,156 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:54,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:54,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:54,157 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:54,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:54,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49bf2290, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:54,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:54,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:54,159 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aad0f1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:54,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:54,160 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:54,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:54,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:54,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2175be9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:54,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:54,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:54,162 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a7c486c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:54,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:54,164 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f59d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:54,165 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:54,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70dae21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:54,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:54,167 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78702107, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:54,168 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f85cb62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:54,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:54,174 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25568c5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:54,176 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c8911c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:54,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:54,178 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4976b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:54,180 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d92b282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:54,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:54,182 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df1119a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:54,183 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67829c62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:54,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:54,185 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ded0a3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:54,187 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d48b167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:54,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:54,188 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c2b393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:54,190 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b06bee8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a84bcce7bad,37069,-1 for getting cluster id 2024-11-09T06:17:54,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T06:17:54,192 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '651e661e-20b2-435b-964a-2cfde8619a3d' 2024-11-09T06:17:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T06:17:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "651e661e-20b2-435b-964a-2cfde8619a3d" 2024-11-09T06:17:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47299202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a84bcce7bad,37069,-1] 2024-11-09T06:17:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T06:17:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:17:54,194 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T06:17:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29296e54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T06:17:54,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:54,197 DEBUG [Thread-2486 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:54,197 DEBUG [Thread-2490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:54,198 DEBUG [Thread-2484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-09T06:17:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:54,199 DEBUG [Thread-2475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:54,199 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T06:17:54,199 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:54,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:54,200 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:54,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:54,202 DEBUG [Thread-2482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:54,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:54,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:54,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:54,204 DEBUG [Thread-2479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:54,204 DEBUG [Thread-2471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:54,205 DEBUG [Thread-2471 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:54,205 DEBUG [Thread-2471 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:54,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:54,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:54,210 DEBUG [Thread-2488 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:54,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:54,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T06:17:54,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:17:54,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:54,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:17:54,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:54,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:17:54,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:54,228 DEBUG [Thread-2473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:54,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:54,239 DEBUG [Thread-2477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T06:17:54,241 DEBUG [Thread-2477 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a84bcce7bad,46317,1731132929173, seqNum=-1] 2024-11-09T06:17:54,241 DEBUG [Thread-2477 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T06:17:54,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T06:17:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5] 2024-11-09T06:17:54,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099fff183fbc744c07ab09d933f2a4782e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_1/A:col10/1731133074217/Put/seqid=0 2024-11-09T06:17:54,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133074238, max timestamp=1731133074268Num uniques 859; 2024-11-09T06:17:54,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:17:54,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133074238, max timestamp=1731133074268Num uniques 859; 2024-11-09T06:17:54,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:17:54,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731133074238, max timestamp=1731133074269Num uniques 859; 2024-11-09T06:17:54,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T06:17:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T06:17:54,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742382_1558 (size=9714) 2024-11-09T06:17:54,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9974, totalHeapSize=22912, min timestamp=1731133074296, max timestamp=1731133074311Num uniques 1018; 2024-11-09T06:17:54,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46614, new segment data size=9974, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-09T06:17:54,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731133074296, max timestamp=1731133074311Num uniques 1018; 2024-11-09T06:17:54,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-09T06:17:54,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731133074296, max timestamp=1731133074310Num uniques 1018; 2024-11-09T06:17:54,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-09T06:17:54,316 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:54,320 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099fff183fbc744c07ab09d933f2a4782e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099fff183fbc744c07ab09d933f2a4782e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,321 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/89b73bf2881d4eb4aeec9a548f9dd28e, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:54,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/89b73bf2881d4eb4aeec9a548f9dd28e is 175, key is test_row_1/A:col10/1731133074217/Put/seqid=0 2024-11-09T06:17:54,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133134337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133134338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133134339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133134341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133134343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T06:17:54,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:54,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742383_1559 (size=22361) 2024-11-09T06:17:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133134452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133134453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133134453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133134453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133134453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T06:17:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T06:17:54,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T06:17:54,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133134663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133134663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133134663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133134664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133134664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,759 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/89b73bf2881d4eb4aeec9a548f9dd28e 2024-11-09T06:17:54,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/3f6ab4c778b945ba8aed0919558363c7 is 50, key is test_row_1/B:col10/1731133074217/Put/seqid=0 2024-11-09T06:17:54,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742384_1560 (size=9657) 2024-11-09T06:17:54,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T06:17:54,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:54,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:54,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/3f6ab4c778b945ba8aed0919558363c7 2024-11-09T06:17:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T06:17:54,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/d2b6fd5f015943d2ba594900d2f46b04 is 50, key is test_row_1/C:col10/1731133074217/Put/seqid=0 2024-11-09T06:17:54,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742385_1561 (size=9657) 2024-11-09T06:17:54,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/d2b6fd5f015943d2ba594900d2f46b04 2024-11-09T06:17:54,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/89b73bf2881d4eb4aeec9a548f9dd28e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/89b73bf2881d4eb4aeec9a548f9dd28e 2024-11-09T06:17:54,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/89b73bf2881d4eb4aeec9a548f9dd28e, entries=100, sequenceid=14, filesize=21.8 K 2024-11-09T06:17:54,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/3f6ab4c778b945ba8aed0919558363c7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3f6ab4c778b945ba8aed0919558363c7 2024-11-09T06:17:54,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3f6ab4c778b945ba8aed0919558363c7, entries=100, sequenceid=14, filesize=9.4 K 2024-11-09T06:17:54,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/d2b6fd5f015943d2ba594900d2f46b04 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/d2b6fd5f015943d2ba594900d2f46b04 2024-11-09T06:17:54,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/d2b6fd5f015943d2ba594900d2f46b04, entries=100, sequenceid=14, filesize=9.4 K 2024-11-09T06:17:54,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 710ms, sequenceid=14, compaction requested=false 2024-11-09T06:17:54,930 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T06:17:54,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:54,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T06:17:54,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:54,964 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=134.18 KB heapSize=379.45 KB 2024-11-09T06:17:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:17:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:17:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:17:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109297dacd9cfdf446997fc62f7b1f235d7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133074333/Put/seqid=0 2024-11-09T06:17:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:54,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742386_1562 (size=12154) 2024-11-09T06:17:54,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133134978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133134977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,985 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109297dacd9cfdf446997fc62f7b1f235d7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109297dacd9cfdf446997fc62f7b1f235d7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/8c7c35d7c421402cb4c6ee33b99c624f, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/8c7c35d7c421402cb4c6ee33b99c624f is 175, key is test_row_0/A:col10/1731133074333/Put/seqid=0 2024-11-09T06:17:54,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133134983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133134984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:54,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133134984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:54,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742387_1563 (size=30955) 2024-11-09T06:17:55,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133135093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133135093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133135093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133135093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133135103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133135302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133135303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133135303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133135303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133135314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T06:17:55,395 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=67, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/8c7c35d7c421402cb4c6ee33b99c624f 2024-11-09T06:17:55,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/673a99b250b5427bb5169ae286aa85cf is 50, key is test_row_0/B:col10/1731133074333/Put/seqid=0 2024-11-09T06:17:55,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742388_1564 (size=12001) 2024-11-09T06:17:55,432 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/673a99b250b5427bb5169ae286aa85cf 2024-11-09T06:17:55,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/5aaa61d3d0fa493a9400bd32c0874e9b is 50, key is test_row_0/C:col10/1731133074333/Put/seqid=0 2024-11-09T06:17:55,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742389_1565 (size=12001) 2024-11-09T06:17:55,455 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/5aaa61d3d0fa493a9400bd32c0874e9b 2024-11-09T06:17:55,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/8c7c35d7c421402cb4c6ee33b99c624f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8c7c35d7c421402cb4c6ee33b99c624f 2024-11-09T06:17:55,467 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8c7c35d7c421402cb4c6ee33b99c624f, entries=150, sequenceid=67, filesize=30.2 K 2024-11-09T06:17:55,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/673a99b250b5427bb5169ae286aa85cf as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/673a99b250b5427bb5169ae286aa85cf 2024-11-09T06:17:55,471 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/673a99b250b5427bb5169ae286aa85cf, entries=150, sequenceid=67, filesize=11.7 K 2024-11-09T06:17:55,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/5aaa61d3d0fa493a9400bd32c0874e9b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5aaa61d3d0fa493a9400bd32c0874e9b 2024-11-09T06:17:55,476 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5aaa61d3d0fa493a9400bd32c0874e9b, entries=150, sequenceid=67, filesize=11.7 K 2024-11-09T06:17:55,478 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~379.41 KB/388512, currentSize=53.67 KB/54960 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 513ms, sequenceid=67, compaction requested=false 2024-11-09T06:17:55,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:55,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:55,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-09T06:17:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-09T06:17:55,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-09T06:17:55,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2800 sec 2024-11-09T06:17:55,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.2840 sec 2024-11-09T06:17:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:55,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T06:17:55,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:17:55,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:55,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:17:55,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:55,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:17:55,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:55,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e8edf643fd7548cfa178e08f7465b76c_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133074976/Put/seqid=0 2024-11-09T06:17:55,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742390_1566 (size=12154) 2024-11-09T06:17:55,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133135635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,642 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133135638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133135639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,646 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e8edf643fd7548cfa178e08f7465b76c_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e8edf643fd7548cfa178e08f7465b76c_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:55,646 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/5e715a336937476a9dac47598d8bad3a, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:55,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/5e715a336937476a9dac47598d8bad3a is 175, key is test_row_0/A:col10/1731133074976/Put/seqid=0 2024-11-09T06:17:55,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133135641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133135642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742391_1567 (size=30955) 2024-11-09T06:17:55,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/5e715a336937476a9dac47598d8bad3a 2024-11-09T06:17:55,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/5231459abda84366888a230e010c1fc7 is 50, key is test_row_0/B:col10/1731133074976/Put/seqid=0 2024-11-09T06:17:55,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742392_1568 (size=12001) 2024-11-09T06:17:55,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/5231459abda84366888a230e010c1fc7 2024-11-09T06:17:55,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/1bfbd64c79d748a5bc774ef96c6a4182 is 50, key is test_row_0/C:col10/1731133074976/Put/seqid=0 2024-11-09T06:17:55,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742393_1569 (size=12001) 2024-11-09T06:17:55,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133135752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133135752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133135752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133135752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133135752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,817 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T06:17:55,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133135963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133135968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133135968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133135968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133135968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/1bfbd64c79d748a5bc774ef96c6a4182 2024-11-09T06:17:56,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/5e715a336937476a9dac47598d8bad3a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5e715a336937476a9dac47598d8bad3a 2024-11-09T06:17:56,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5e715a336937476a9dac47598d8bad3a, entries=150, sequenceid=79, filesize=30.2 K 2024-11-09T06:17:56,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/5231459abda84366888a230e010c1fc7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5231459abda84366888a230e010c1fc7 2024-11-09T06:17:56,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5231459abda84366888a230e010c1fc7, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T06:17:56,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/1bfbd64c79d748a5bc774ef96c6a4182 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/1bfbd64c79d748a5bc774ef96c6a4182 2024-11-09T06:17:56,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/1bfbd64c79d748a5bc774ef96c6a4182, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T06:17:56,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 495ms, sequenceid=79, compaction requested=true 2024-11-09T06:17:56,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:56,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:56,111 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:56,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:56,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:56,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:56,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:56,111 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:56,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:56,112 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84271 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:56,112 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:56,112 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B is initiating minor compaction (all files) 2024-11-09T06:17:56,112 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A is initiating minor compaction (all files) 2024-11-09T06:17:56,112 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,112 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,112 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3f6ab4c778b945ba8aed0919558363c7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/673a99b250b5427bb5169ae286aa85cf, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5231459abda84366888a230e010c1fc7] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=32.9 K 2024-11-09T06:17:56,112 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/89b73bf2881d4eb4aeec9a548f9dd28e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8c7c35d7c421402cb4c6ee33b99c624f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5e715a336937476a9dac47598d8bad3a] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=82.3 K 2024-11-09T06:17:56,112 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,112 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/89b73bf2881d4eb4aeec9a548f9dd28e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8c7c35d7c421402cb4c6ee33b99c624f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5e715a336937476a9dac47598d8bad3a] 2024-11-09T06:17:56,113 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f6ab4c778b945ba8aed0919558363c7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731133074217 2024-11-09T06:17:56,113 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89b73bf2881d4eb4aeec9a548f9dd28e, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731133074217 2024-11-09T06:17:56,113 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c7c35d7c421402cb4c6ee33b99c624f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731133074333 2024-11-09T06:17:56,113 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 673a99b250b5427bb5169ae286aa85cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731133074333 2024-11-09T06:17:56,113 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e715a336937476a9dac47598d8bad3a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731133074976 2024-11-09T06:17:56,113 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5231459abda84366888a230e010c1fc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731133074976 2024-11-09T06:17:56,123 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:56,138 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109d1f3c581a093485bb05f9950b9b81171_1aeacadfe7a3b9ebc8c8b0e6e39a10ca store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:56,140 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109d1f3c581a093485bb05f9950b9b81171_1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:56,141 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d1f3c581a093485bb05f9950b9b81171_1aeacadfe7a3b9ebc8c8b0e6e39a10ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:56,141 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#B#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:56,141 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/165d4bd0a1de4e7098c0fb78feaa8f5e is 50, key is test_row_0/B:col10/1731133074976/Put/seqid=0 2024-11-09T06:17:56,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742395_1571 (size=12104) 2024-11-09T06:17:56,181 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/165d4bd0a1de4e7098c0fb78feaa8f5e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/165d4bd0a1de4e7098c0fb78feaa8f5e 2024-11-09T06:17:56,189 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 165d4bd0a1de4e7098c0fb78feaa8f5e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:56,189 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:56,189 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B, priority=13, startTime=1731133076111; duration=0sec 2024-11-09T06:17:56,189 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:56,189 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B 2024-11-09T06:17:56,189 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:56,190 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:56,190 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C is initiating minor compaction (all files) 2024-11-09T06:17:56,190 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,190 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/d2b6fd5f015943d2ba594900d2f46b04, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5aaa61d3d0fa493a9400bd32c0874e9b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/1bfbd64c79d748a5bc774ef96c6a4182] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=32.9 K 2024-11-09T06:17:56,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742394_1570 (size=4469) 2024-11-09T06:17:56,191 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting d2b6fd5f015943d2ba594900d2f46b04, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731133074217 2024-11-09T06:17:56,191 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aaa61d3d0fa493a9400bd32c0874e9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731133074333 2024-11-09T06:17:56,192 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bfbd64c79d748a5bc774ef96c6a4182, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731133074976 2024-11-09T06:17:56,194 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#A#compaction#487 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:56,194 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/46e21a9f09ed4c6f9d3fa61e3b722cae is 175, key is test_row_0/A:col10/1731133074976/Put/seqid=0 2024-11-09T06:17:56,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742396_1572 (size=31058) 2024-11-09T06:17:56,209 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:56,209 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/1c54e2745eaf4178a680fd4aa5d3eced is 50, key is test_row_0/C:col10/1731133074976/Put/seqid=0 2024-11-09T06:17:56,216 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/46e21a9f09ed4c6f9d3fa61e3b722cae as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/46e21a9f09ed4c6f9d3fa61e3b722cae 2024-11-09T06:17:56,220 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 46e21a9f09ed4c6f9d3fa61e3b722cae(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:56,220 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:56,220 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A, priority=13, startTime=1731133076111; duration=0sec 2024-11-09T06:17:56,220 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:56,220 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A 2024-11-09T06:17:56,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742397_1573 (size=12104) 2024-11-09T06:17:56,243 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/1c54e2745eaf4178a680fd4aa5d3eced as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/1c54e2745eaf4178a680fd4aa5d3eced 2024-11-09T06:17:56,249 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 1c54e2745eaf4178a680fd4aa5d3eced(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:56,250 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:56,250 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C, priority=13, startTime=1731133076111; duration=0sec 2024-11-09T06:17:56,250 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:56,250 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C 2024-11-09T06:17:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:56,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-09T06:17:56,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:17:56,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:17:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:17:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:56,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110932b0e752149b4455ad6bcf1efcec64a8_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133076275/Put/seqid=0 2024-11-09T06:17:56,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133136293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133136292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133136294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133136301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133136301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T06:17:56,343 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:17:56,343 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:17:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-09T06:17:56,345 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:17:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T06:17:56,346 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:17:56,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742398_1574 (size=12154) 2024-11-09T06:17:56,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:17:56,347 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:56,351 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110932b0e752149b4455ad6bcf1efcec64a8_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110932b0e752149b4455ad6bcf1efcec64a8_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:56,352 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/7413fa0178b84aa4a55563b7600639d5, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:56,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/7413fa0178b84aa4a55563b7600639d5 is 175, key is test_row_0/A:col10/1731133076275/Put/seqid=0 2024-11-09T06:17:56,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742399_1575 (size=30955) 2024-11-09T06:17:56,365 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/7413fa0178b84aa4a55563b7600639d5 2024-11-09T06:17:56,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/36dfd8556eed42dd979c730f91df57d6 is 50, key is test_row_0/B:col10/1731133076275/Put/seqid=0 2024-11-09T06:17:56,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742400_1576 (size=12001) 2024-11-09T06:17:56,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/36dfd8556eed42dd979c730f91df57d6 2024-11-09T06:17:56,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/c22f4734187c48d3a17bf008290022a0 is 50, key is test_row_0/C:col10/1731133076275/Put/seqid=0 2024-11-09T06:17:56,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742401_1577 (size=12001) 2024-11-09T06:17:56,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133136413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133136413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133136413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133136413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133136413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T06:17:56,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T06:17:56,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:56,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133136623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133136623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133136623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133136623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133136624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T06:17:56,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:56,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T06:17:56,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/c22f4734187c48d3a17bf008290022a0 2024-11-09T06:17:56,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/7413fa0178b84aa4a55563b7600639d5 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/7413fa0178b84aa4a55563b7600639d5 2024-11-09T06:17:56,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/7413fa0178b84aa4a55563b7600639d5, entries=150, sequenceid=103, filesize=30.2 K 2024-11-09T06:17:56,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/36dfd8556eed42dd979c730f91df57d6 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/36dfd8556eed42dd979c730f91df57d6 2024-11-09T06:17:56,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T06:17:56,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:56,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/36dfd8556eed42dd979c730f91df57d6, entries=150, sequenceid=103, filesize=11.7 K 2024-11-09T06:17:56,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/c22f4734187c48d3a17bf008290022a0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c22f4734187c48d3a17bf008290022a0 2024-11-09T06:17:56,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c22f4734187c48d3a17bf008290022a0, entries=150, sequenceid=103, filesize=11.7 K 2024-11-09T06:17:56,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 547ms, sequenceid=103, compaction requested=false 2024-11-09T06:17:56,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:56,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T06:17:56,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:17:56,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:56,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:17:56,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:56,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:17:56,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:56,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ded96b8194514d3dae0add5703f6d98e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133076933/Put/seqid=0 2024-11-09T06:17:56,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T06:17:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:56,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:56,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133136961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133136961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133076955, max timestamp=1731133076966Num uniques 859; 2024-11-09T06:17:56,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:56,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731133076955, max timestamp=1731133076966Num uniques 859; 2024-11-09T06:17:56,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4a84bcce7bad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T06:17:56,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742402_1578 (size=12154) 2024-11-09T06:17:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T06:17:56,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133136983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133136985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:56,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133136988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:56,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133137072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133137072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133137093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133137093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133137103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T06:17:57,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:57,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:57,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:57,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:57,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:57,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T06:17:57,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:57,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:57,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:57,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:57,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:57,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133137283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133137283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,300 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T06:17:57,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133137303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133137304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133137313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,373 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:57,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ded96b8194514d3dae0add5703f6d98e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ded96b8194514d3dae0add5703f6d98e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:57,379 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/4cdbb0bc1801477a8ed0b37e36b33fe3, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:57,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/4cdbb0bc1801477a8ed0b37e36b33fe3 is 175, key is test_row_0/A:col10/1731133076933/Put/seqid=0 2024-11-09T06:17:57,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742403_1579 (size=30955) 2024-11-09T06:17:57,390 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/4cdbb0bc1801477a8ed0b37e36b33fe3 2024-11-09T06:17:57,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/a1894d2139c64161aa20bcb7689c910f is 50, key is test_row_0/B:col10/1731133076933/Put/seqid=0 2024-11-09T06:17:57,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T06:17:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:57,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:57,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742404_1580 (size=12001) 2024-11-09T06:17:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:17:57,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/a1894d2139c64161aa20bcb7689c910f 2024-11-09T06:17:57,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/0fb9e7d6d6a241738f13bbef99090d2f is 50, key is test_row_0/C:col10/1731133076933/Put/seqid=0 2024-11-09T06:17:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T06:17:57,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742405_1581 (size=12001) 2024-11-09T06:17:57,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/0fb9e7d6d6a241738f13bbef99090d2f 2024-11-09T06:17:57,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/4cdbb0bc1801477a8ed0b37e36b33fe3 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4cdbb0bc1801477a8ed0b37e36b33fe3 2024-11-09T06:17:57,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4cdbb0bc1801477a8ed0b37e36b33fe3, entries=150, sequenceid=115, filesize=30.2 K 2024-11-09T06:17:57,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/a1894d2139c64161aa20bcb7689c910f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a1894d2139c64161aa20bcb7689c910f 2024-11-09T06:17:57,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a1894d2139c64161aa20bcb7689c910f, entries=150, sequenceid=115, filesize=11.7 K 2024-11-09T06:17:57,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/0fb9e7d6d6a241738f13bbef99090d2f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/0fb9e7d6d6a241738f13bbef99090d2f 2024-11-09T06:17:57,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/0fb9e7d6d6a241738f13bbef99090d2f, entries=150, sequenceid=115, filesize=11.7 K 2024-11-09T06:17:57,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=116.29 KB/119080 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 586ms, sequenceid=115, compaction requested=true 2024-11-09T06:17:57,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:57,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:57,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:57,521 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:57,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:57,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:57,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:57,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T06:17:57,521 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:57,522 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:57,522 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A is initiating minor compaction (all files) 2024-11-09T06:17:57,522 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:57,522 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/46e21a9f09ed4c6f9d3fa61e3b722cae, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/7413fa0178b84aa4a55563b7600639d5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4cdbb0bc1801477a8ed0b37e36b33fe3] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=90.8 K 2024-11-09T06:17:57,522 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:57,522 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/46e21a9f09ed4c6f9d3fa61e3b722cae, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/7413fa0178b84aa4a55563b7600639d5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4cdbb0bc1801477a8ed0b37e36b33fe3] 2024-11-09T06:17:57,523 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:57,523 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46e21a9f09ed4c6f9d3fa61e3b722cae, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731133074976 2024-11-09T06:17:57,523 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B is initiating minor compaction (all files) 2024-11-09T06:17:57,523 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:57,523 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/165d4bd0a1de4e7098c0fb78feaa8f5e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/36dfd8556eed42dd979c730f91df57d6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a1894d2139c64161aa20bcb7689c910f] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=35.3 K 2024-11-09T06:17:57,523 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7413fa0178b84aa4a55563b7600639d5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731133075637 2024-11-09T06:17:57,523 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 165d4bd0a1de4e7098c0fb78feaa8f5e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731133074976 2024-11-09T06:17:57,523 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cdbb0bc1801477a8ed0b37e36b33fe3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731133076278 2024-11-09T06:17:57,524 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 36dfd8556eed42dd979c730f91df57d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731133075637 2024-11-09T06:17:57,524 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a1894d2139c64161aa20bcb7689c910f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731133076278 2024-11-09T06:17:57,533 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:57,535 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#B#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:57,536 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/acd4057451964ec783644c39368b17a8 is 50, key is test_row_0/B:col10/1731133076933/Put/seqid=0 2024-11-09T06:17:57,537 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411093bba6009ddd8444d92f4755a07fbd1c8_1aeacadfe7a3b9ebc8c8b0e6e39a10ca store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:57,539 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411093bba6009ddd8444d92f4755a07fbd1c8_1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:57,540 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093bba6009ddd8444d92f4755a07fbd1c8_1aeacadfe7a3b9ebc8c8b0e6e39a10ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:57,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742406_1582 (size=12207) 2024-11-09T06:17:57,546 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/acd4057451964ec783644c39368b17a8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/acd4057451964ec783644c39368b17a8 2024-11-09T06:17:57,551 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into acd4057451964ec783644c39368b17a8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:57,551 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:57,551 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B, priority=13, startTime=1731133077521; duration=0sec 2024-11-09T06:17:57,551 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:57,551 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B 2024-11-09T06:17:57,551 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:57,552 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:57,552 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C is initiating minor compaction (all files) 2024-11-09T06:17:57,552 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:57,552 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/1c54e2745eaf4178a680fd4aa5d3eced, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c22f4734187c48d3a17bf008290022a0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/0fb9e7d6d6a241738f13bbef99090d2f] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=35.3 K 2024-11-09T06:17:57,553 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c54e2745eaf4178a680fd4aa5d3eced, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731133074976 2024-11-09T06:17:57,553 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting c22f4734187c48d3a17bf008290022a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731133075637 2024-11-09T06:17:57,553 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fb9e7d6d6a241738f13bbef99090d2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731133076278 2024-11-09T06:17:57,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742407_1583 (size=4469) 2024-11-09T06:17:57,572 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#A#compaction#496 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:57,572 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/acc6f9c1983f47e5b4049aee42356f6c is 175, key is test_row_0/A:col10/1731133076933/Put/seqid=0 2024-11-09T06:17:57,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T06:17:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:57,587 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=116.29 KB heapSize=333.70 KB 2024-11-09T06:17:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:17:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:17:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:17:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T06:17:57,587 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:57,588 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/cfc4f5e162774545aef95f54725163e5 is 50, key is test_row_0/C:col10/1731133076933/Put/seqid=0 2024-11-09T06:17:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:57,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:17:57,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742408_1584 (size=31161) 2024-11-09T06:17:57,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093e84c74da477443b91001c58809124b2_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133076979/Put/seqid=0 2024-11-09T06:17:57,620 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/acc6f9c1983f47e5b4049aee42356f6c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/acc6f9c1983f47e5b4049aee42356f6c 2024-11-09T06:17:57,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133137618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,626 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into acc6f9c1983f47e5b4049aee42356f6c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:57,626 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:57,626 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A, priority=13, startTime=1731133077521; duration=0sec 2024-11-09T06:17:57,626 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:57,626 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A 2024-11-09T06:17:57,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133137621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742409_1585 (size=12207) 2024-11-09T06:17:57,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133137622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133137623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133137623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742410_1586 (size=12304) 2024-11-09T06:17:57,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133137733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133137733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133137733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133137943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133137943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133137943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:58,037 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/cfc4f5e162774545aef95f54725163e5 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/cfc4f5e162774545aef95f54725163e5 2024-11-09T06:17:58,043 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into cfc4f5e162774545aef95f54725163e5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:58,043 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:58,043 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C, priority=13, startTime=1731133077521; duration=0sec 2024-11-09T06:17:58,043 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:58,043 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C 2024-11-09T06:17:58,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:58,051 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093e84c74da477443b91001c58809124b2_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093e84c74da477443b91001c58809124b2_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/9f0bc5a9027448c6ba6ad767b5ffc751, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/9f0bc5a9027448c6ba6ad767b5ffc751 is 175, key is test_row_0/A:col10/1731133076979/Put/seqid=0 2024-11-09T06:17:58,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742411_1587 (size=31105) 2024-11-09T06:17:58,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:58,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133138133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133138133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133138252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133138253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:58,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133138263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:58,467 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/9f0bc5a9027448c6ba6ad767b5ffc751 2024-11-09T06:17:58,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/52f78ad049c440a8b37feba117a18e34 is 50, key is test_row_0/B:col10/1731133076979/Put/seqid=0 2024-11-09T06:17:58,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742412_1588 (size=12151) 2024-11-09T06:17:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T06:17:58,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133138762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:58,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133138762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:58,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133138773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:58,878 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/52f78ad049c440a8b37feba117a18e34 2024-11-09T06:17:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/89298c053b724edc98f5d336b597d053 is 50, key is test_row_0/C:col10/1731133076979/Put/seqid=0 2024-11-09T06:17:58,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742413_1589 (size=12151) 2024-11-09T06:17:58,958 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/89298c053b724edc98f5d336b597d053 2024-11-09T06:17:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/9f0bc5a9027448c6ba6ad767b5ffc751 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/9f0bc5a9027448c6ba6ad767b5ffc751 2024-11-09T06:17:58,982 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/9f0bc5a9027448c6ba6ad767b5ffc751, entries=150, sequenceid=145, filesize=30.4 K 2024-11-09T06:17:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/52f78ad049c440a8b37feba117a18e34 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/52f78ad049c440a8b37feba117a18e34 2024-11-09T06:17:58,987 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/52f78ad049c440a8b37feba117a18e34, entries=150, sequenceid=145, filesize=11.9 K 2024-11-09T06:17:58,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/89298c053b724edc98f5d336b597d053 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/89298c053b724edc98f5d336b597d053 2024-11-09T06:17:58,993 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/89298c053b724edc98f5d336b597d053, entries=150, sequenceid=145, filesize=11.9 K 2024-11-09T06:17:58,995 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~116.29 KB/119080, heapSize ~333.66 KB/341664, currentSize=60.38 KB/61830 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 1409ms, sequenceid=145, compaction requested=false 2024-11-09T06:17:58,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:58,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:58,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-09T06:17:58,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-09T06:17:59,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-09T06:17:59,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6520 sec 2024-11-09T06:17:59,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.6620 sec 2024-11-09T06:17:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:59,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:17:59,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:17:59,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:59,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:17:59,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:59,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:17:59,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:59,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109eb634e11fe424da18403b26933ff96ac_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133079153/Put/seqid=0 2024-11-09T06:17:59,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742414_1590 (size=12304) 2024-11-09T06:17:59,170 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:59,176 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109eb634e11fe424da18403b26933ff96ac_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eb634e11fe424da18403b26933ff96ac_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:59,177 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/afe90f4d727f423ba5ed45d384819de4, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:59,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/afe90f4d727f423ba5ed45d384819de4 is 175, key is test_row_0/A:col10/1731133079153/Put/seqid=0 2024-11-09T06:17:59,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742415_1591 (size=31105) 2024-11-09T06:17:59,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133139202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133139207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133139314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133139314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133139524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133139524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,587 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/afe90f4d727f423ba5ed45d384819de4 2024-11-09T06:17:59,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/32cadad2c1b244bc91bde122d6ad805c is 50, key is test_row_0/B:col10/1731133079153/Put/seqid=0 2024-11-09T06:17:59,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742416_1592 (size=12151) 2024-11-09T06:17:59,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/32cadad2c1b244bc91bde122d6ad805c 2024-11-09T06:17:59,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/7b52d1af95344543bda8122cc1ca41af is 50, key is test_row_0/C:col10/1731133079153/Put/seqid=0 2024-11-09T06:17:59,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742417_1593 (size=12151) 2024-11-09T06:17:59,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/7b52d1af95344543bda8122cc1ca41af 2024-11-09T06:17:59,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/afe90f4d727f423ba5ed45d384819de4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/afe90f4d727f423ba5ed45d384819de4 2024-11-09T06:17:59,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/afe90f4d727f423ba5ed45d384819de4, entries=150, sequenceid=161, filesize=30.4 K 2024-11-09T06:17:59,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/32cadad2c1b244bc91bde122d6ad805c as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/32cadad2c1b244bc91bde122d6ad805c 2024-11-09T06:17:59,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/32cadad2c1b244bc91bde122d6ad805c, entries=150, sequenceid=161, filesize=11.9 K 2024-11-09T06:17:59,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/7b52d1af95344543bda8122cc1ca41af as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7b52d1af95344543bda8122cc1ca41af 2024-11-09T06:17:59,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7b52d1af95344543bda8122cc1ca41af, entries=150, sequenceid=161, filesize=11.9 K 2024-11-09T06:17:59,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 521ms, sequenceid=161, compaction requested=true 2024-11-09T06:17:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:17:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:59,675 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:59,675 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:17:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:17:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:59,677 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93371 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:59,677 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A is initiating minor compaction (all files) 2024-11-09T06:17:59,677 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:59,677 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/acc6f9c1983f47e5b4049aee42356f6c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/9f0bc5a9027448c6ba6ad767b5ffc751, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/afe90f4d727f423ba5ed45d384819de4] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=91.2 K 2024-11-09T06:17:59,677 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:59,677 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/acc6f9c1983f47e5b4049aee42356f6c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/9f0bc5a9027448c6ba6ad767b5ffc751, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/afe90f4d727f423ba5ed45d384819de4] 2024-11-09T06:17:59,677 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:59,677 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B is initiating minor compaction (all files) 2024-11-09T06:17:59,677 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:59,678 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/acd4057451964ec783644c39368b17a8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/52f78ad049c440a8b37feba117a18e34, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/32cadad2c1b244bc91bde122d6ad805c] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=35.7 K 2024-11-09T06:17:59,678 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting acc6f9c1983f47e5b4049aee42356f6c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731133076278 2024-11-09T06:17:59,678 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting acd4057451964ec783644c39368b17a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731133076278 2024-11-09T06:17:59,678 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f0bc5a9027448c6ba6ad767b5ffc751, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731133076964 2024-11-09T06:17:59,678 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 52f78ad049c440a8b37feba117a18e34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731133076964 2024-11-09T06:17:59,678 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting afe90f4d727f423ba5ed45d384819de4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731133077609 2024-11-09T06:17:59,678 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 32cadad2c1b244bc91bde122d6ad805c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731133077609 2024-11-09T06:17:59,687 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:59,688 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#B#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:59,688 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/2938f6d80c2d40f7ade1a3566223f167 is 50, key is test_row_0/B:col10/1731133079153/Put/seqid=0 2024-11-09T06:17:59,701 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109d05199bc4d494b74bd1f1e3bde77e543_1aeacadfe7a3b9ebc8c8b0e6e39a10ca store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:59,703 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109d05199bc4d494b74bd1f1e3bde77e543_1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:59,703 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d05199bc4d494b74bd1f1e3bde77e543_1aeacadfe7a3b9ebc8c8b0e6e39a10ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:59,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742418_1594 (size=12459) 2024-11-09T06:17:59,736 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/2938f6d80c2d40f7ade1a3566223f167 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/2938f6d80c2d40f7ade1a3566223f167 2024-11-09T06:17:59,742 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 2938f6d80c2d40f7ade1a3566223f167(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:59,742 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:59,742 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B, priority=13, startTime=1731133079675; duration=0sec 2024-11-09T06:17:59,742 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:17:59,742 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B 2024-11-09T06:17:59,742 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:17:59,743 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:17:59,743 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C is initiating minor compaction (all files) 2024-11-09T06:17:59,743 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:17:59,743 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/cfc4f5e162774545aef95f54725163e5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/89298c053b724edc98f5d336b597d053, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7b52d1af95344543bda8122cc1ca41af] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=35.7 K 2024-11-09T06:17:59,743 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting cfc4f5e162774545aef95f54725163e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731133076278 2024-11-09T06:17:59,743 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 89298c053b724edc98f5d336b597d053, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731133076964 2024-11-09T06:17:59,744 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b52d1af95344543bda8122cc1ca41af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731133077609 2024-11-09T06:17:59,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742419_1595 (size=4469) 2024-11-09T06:17:59,747 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#A#compaction#506 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:59,748 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/465dcbc377194856806773d5b1c4d295 is 175, key is test_row_0/A:col10/1731133079153/Put/seqid=0 2024-11-09T06:17:59,757 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:17:59,757 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/7b90fe6d725b4cf5aaad058f6f773048 is 50, key is test_row_0/C:col10/1731133079153/Put/seqid=0 2024-11-09T06:17:59,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742420_1596 (size=31413) 2024-11-09T06:17:59,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742421_1597 (size=12459) 2024-11-09T06:17:59,776 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/7b90fe6d725b4cf5aaad058f6f773048 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7b90fe6d725b4cf5aaad058f6f773048 2024-11-09T06:17:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:59,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T06:17:59,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:17:59,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:59,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:17:59,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:59,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:17:59,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:17:59,782 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 7b90fe6d725b4cf5aaad058f6f773048(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:17:59,782 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:17:59,782 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C, priority=13, startTime=1731133079676; duration=0sec 2024-11-09T06:17:59,782 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:17:59,782 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C 2024-11-09T06:17:59,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099732a93a6427474baba7f92023cd730b_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133079775/Put/seqid=0 2024-11-09T06:17:59,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742422_1598 (size=12304) 2024-11-09T06:17:59,795 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:17:59,798 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099732a93a6427474baba7f92023cd730b_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099732a93a6427474baba7f92023cd730b_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:17:59,799 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/1a1c6e8fc01744f8824ed192c216f3c4, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:17:59,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/1a1c6e8fc01744f8824ed192c216f3c4 is 175, key is test_row_0/A:col10/1731133079775/Put/seqid=0 2024-11-09T06:17:59,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742423_1599 (size=31105) 2024-11-09T06:17:59,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133139803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133139805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133139806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133139833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133139833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133139912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133139912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:17:59,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:17:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133139923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:17:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:17:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133140122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:00,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133140122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133140133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,165 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/465dcbc377194856806773d5b1c4d295 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/465dcbc377194856806773d5b1c4d295 2024-11-09T06:18:00,168 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 465dcbc377194856806773d5b1c4d295(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:00,168 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:00,168 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A, priority=13, startTime=1731133079675; duration=0sec 2024-11-09T06:18:00,168 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:00,168 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A 2024-11-09T06:18:00,205 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/1a1c6e8fc01744f8824ed192c216f3c4 2024-11-09T06:18:00,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/928e879bdca5434b8514d05c34fbea77 is 50, key is test_row_0/B:col10/1731133079775/Put/seqid=0 2024-11-09T06:18:00,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742424_1600 (size=12151) 2024-11-09T06:18:00,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133140342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:00,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133140342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133140433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133140433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133140442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T06:18:00,513 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:18:00,514 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:18:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-09T06:18:00,516 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:18:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T06:18:00,517 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:18:00,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:18:00,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/928e879bdca5434b8514d05c34fbea77 2024-11-09T06:18:00,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/7cc163c917c94f87897dab982da693ba is 50, key is test_row_0/C:col10/1731133079775/Put/seqid=0 2024-11-09T06:18:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T06:18:00,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742425_1601 (size=12151) 2024-11-09T06:18:00,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T06:18:00,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:00,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:00,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:00,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:00,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:00,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T06:18:00,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:00,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:00,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:00,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:00,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T06:18:00,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133140943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133140944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:00,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133140954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:00,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T06:18:00,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:00,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:00,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:00,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:01,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/7cc163c917c94f87897dab982da693ba 2024-11-09T06:18:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/1a1c6e8fc01744f8824ed192c216f3c4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1a1c6e8fc01744f8824ed192c216f3c4 2024-11-09T06:18:01,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1a1c6e8fc01744f8824ed192c216f3c4, entries=150, sequenceid=181, filesize=30.4 K 2024-11-09T06:18:01,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/928e879bdca5434b8514d05c34fbea77 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/928e879bdca5434b8514d05c34fbea77 2024-11-09T06:18:01,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/928e879bdca5434b8514d05c34fbea77, entries=150, sequenceid=181, filesize=11.9 K 2024-11-09T06:18:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/7cc163c917c94f87897dab982da693ba as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7cc163c917c94f87897dab982da693ba 2024-11-09T06:18:01,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7cc163c917c94f87897dab982da693ba, entries=150, sequenceid=181, filesize=11.9 K 2024-11-09T06:18:01,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 1284ms, sequenceid=181, compaction requested=false 2024-11-09T06:18:01,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:01,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T06:18:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:01,133 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:18:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T06:18:01,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091c4611e68967405c8bceb5dd2650f500_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133079803/Put/seqid=0 2024-11-09T06:18:01,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742426_1602 (size=12304) 2024-11-09T06:18:01,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:01,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133141419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:01,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133141423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:01,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133141532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:01,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133141532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:01,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:01,575 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091c4611e68967405c8bceb5dd2650f500_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091c4611e68967405c8bceb5dd2650f500_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/cd414e6c8bfa4e26bb72534334456e25, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/cd414e6c8bfa4e26bb72534334456e25 is 175, key is test_row_0/A:col10/1731133079803/Put/seqid=0 2024-11-09T06:18:01,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742427_1603 (size=31105) 2024-11-09T06:18:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T06:18:01,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133141743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:01,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133141743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:01,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:01,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:01,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133141953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:01,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133141953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:01,956 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_0' in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:18:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:01,956 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:18:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:01,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133141972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:01,975 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:18:01,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:01,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:01,980 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/cd414e6c8bfa4e26bb72534334456e25 2024-11-09T06:18:01,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/bde4934ea4c6425ea3b56109acd8c5ea is 50, key is test_row_0/B:col10/1731133079803/Put/seqid=0 2024-11-09T06:18:02,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742428_1604 (size=12151) 2024-11-09T06:18:02,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133142052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:02,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133142052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:02,412 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/bde4934ea4c6425ea3b56109acd8c5ea 2024-11-09T06:18:02,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/a3ad618400e74e318ff5016c903311a8 is 50, key is test_row_0/C:col10/1731133079803/Put/seqid=0 2024-11-09T06:18:02,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742429_1605 (size=12151) 2024-11-09T06:18:02,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133142562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:02,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133142563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T06:18:02,862 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/a3ad618400e74e318ff5016c903311a8 2024-11-09T06:18:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/cd414e6c8bfa4e26bb72534334456e25 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cd414e6c8bfa4e26bb72534334456e25 2024-11-09T06:18:02,891 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cd414e6c8bfa4e26bb72534334456e25, entries=150, sequenceid=196, filesize=30.4 K 2024-11-09T06:18:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/bde4934ea4c6425ea3b56109acd8c5ea as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/bde4934ea4c6425ea3b56109acd8c5ea 2024-11-09T06:18:02,906 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/bde4934ea4c6425ea3b56109acd8c5ea, entries=150, sequenceid=196, filesize=11.9 K 2024-11-09T06:18:02,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/a3ad618400e74e318ff5016c903311a8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a3ad618400e74e318ff5016c903311a8 2024-11-09T06:18:02,911 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a3ad618400e74e318ff5016c903311a8, entries=150, sequenceid=196, filesize=11.9 K 2024-11-09T06:18:02,913 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 1781ms, sequenceid=196, compaction requested=true 2024-11-09T06:18:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-09T06:18:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-09T06:18:02,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-09T06:18:02,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4000 sec 2024-11-09T06:18:02,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.4100 sec 2024-11-09T06:18:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:03,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:18:03,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:03,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:03,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:03,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:03,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:03,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:03,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dbb6eef2486840d99a25dd40394eaae7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133083574/Put/seqid=0 2024-11-09T06:18:03,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133143618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:03,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133143625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:03,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:03,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742430_1606 (size=14794) 2024-11-09T06:18:03,643 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:03,647 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dbb6eef2486840d99a25dd40394eaae7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dbb6eef2486840d99a25dd40394eaae7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:03,649 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/180a6d75d42546db92a00bff108a2af9, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:03,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/180a6d75d42546db92a00bff108a2af9 is 175, key is test_row_0/A:col10/1731133083574/Put/seqid=0 2024-11-09T06:18:03,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742431_1607 (size=39749) 2024-11-09T06:18:03,678 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/180a6d75d42546db92a00bff108a2af9 2024-11-09T06:18:03,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/1deb2698a61e487386ff1342d187d506 is 50, key is test_row_0/B:col10/1731133083574/Put/seqid=0 2024-11-09T06:18:03,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742432_1608 (size=12151) 2024-11-09T06:18:03,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/1deb2698a61e487386ff1342d187d506 2024-11-09T06:18:03,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133143734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:03,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:03,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133143734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:03,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:03,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:03,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/a0533f4dac9340348011268168e04f16 is 50, key is test_row_0/C:col10/1731133083574/Put/seqid=0 2024-11-09T06:18:03,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742433_1609 (size=12151) 2024-11-09T06:18:03,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:03,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133143942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133143942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:03,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133143973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:03,977 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_0' in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:18:03,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133143973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:03,978 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:18:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:03,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133143993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:03,997 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:18:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:04,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/a0533f4dac9340348011268168e04f16 2024-11-09T06:18:04,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/180a6d75d42546db92a00bff108a2af9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/180a6d75d42546db92a00bff108a2af9 2024-11-09T06:18:04,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/180a6d75d42546db92a00bff108a2af9, entries=200, sequenceid=216, filesize=38.8 K 2024-11-09T06:18:04,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/1deb2698a61e487386ff1342d187d506 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/1deb2698a61e487386ff1342d187d506 2024-11-09T06:18:04,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/1deb2698a61e487386ff1342d187d506, entries=150, sequenceid=216, filesize=11.9 K 2024-11-09T06:18:04,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/a0533f4dac9340348011268168e04f16 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a0533f4dac9340348011268168e04f16 2024-11-09T06:18:04,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a0533f4dac9340348011268168e04f16, entries=150, sequenceid=216, filesize=11.9 K 2024-11-09T06:18:04,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 653ms, sequenceid=216, compaction requested=true 2024-11-09T06:18:04,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:04,229 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:18:04,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:18:04,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:04,230 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:18:04,230 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133372 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:18:04,230 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A is initiating minor compaction (all files) 2024-11-09T06:18:04,230 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:04,231 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/465dcbc377194856806773d5b1c4d295, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1a1c6e8fc01744f8824ed192c216f3c4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cd414e6c8bfa4e26bb72534334456e25, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/180a6d75d42546db92a00bff108a2af9] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=130.2 K 2024-11-09T06:18:04,231 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:04,231 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/465dcbc377194856806773d5b1c4d295, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1a1c6e8fc01744f8824ed192c216f3c4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cd414e6c8bfa4e26bb72534334456e25, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/180a6d75d42546db92a00bff108a2af9] 2024-11-09T06:18:04,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:18:04,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:04,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:18:04,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:18:04,231 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 465dcbc377194856806773d5b1c4d295, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731133077609 2024-11-09T06:18:04,231 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1c6e8fc01744f8824ed192c216f3c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731133079199 2024-11-09T06:18:04,232 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:18:04,232 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B is initiating minor compaction (all files) 2024-11-09T06:18:04,232 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:04,232 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/2938f6d80c2d40f7ade1a3566223f167, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/928e879bdca5434b8514d05c34fbea77, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/bde4934ea4c6425ea3b56109acd8c5ea, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/1deb2698a61e487386ff1342d187d506] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=47.8 K 2024-11-09T06:18:04,232 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd414e6c8bfa4e26bb72534334456e25, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731133079794 2024-11-09T06:18:04,233 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 180a6d75d42546db92a00bff108a2af9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731133081389 2024-11-09T06:18:04,233 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2938f6d80c2d40f7ade1a3566223f167, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731133077609 2024-11-09T06:18:04,233 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 928e879bdca5434b8514d05c34fbea77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731133079199 2024-11-09T06:18:04,233 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting bde4934ea4c6425ea3b56109acd8c5ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731133079794 2024-11-09T06:18:04,233 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1deb2698a61e487386ff1342d187d506, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731133081389 2024-11-09T06:18:04,245 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#B#compaction#517 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:04,246 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/aee15b5ec6d54450891ba775c095ec1b is 50, key is test_row_0/B:col10/1731133083574/Put/seqid=0 2024-11-09T06:18:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:04,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:18:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:04,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:04,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:04,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:04,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:04,269 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:04,283 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109b31293f424284dfd8b50fba735bda2de_1aeacadfe7a3b9ebc8c8b0e6e39a10ca store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:04,287 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109b31293f424284dfd8b50fba735bda2de_1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:04,288 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b31293f424284dfd8b50fba735bda2de_1aeacadfe7a3b9ebc8c8b0e6e39a10ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:04,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742434_1610 (size=12595) 2024-11-09T06:18:04,305 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/aee15b5ec6d54450891ba775c095ec1b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/aee15b5ec6d54450891ba775c095ec1b 2024-11-09T06:18:04,309 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into aee15b5ec6d54450891ba775c095ec1b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:04,309 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:04,309 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B, priority=12, startTime=1731133084230; duration=0sec 2024-11-09T06:18:04,309 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:18:04,309 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B 2024-11-09T06:18:04,309 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T06:18:04,310 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T06:18:04,310 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C is initiating minor compaction (all files) 2024-11-09T06:18:04,310 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:04,311 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7b90fe6d725b4cf5aaad058f6f773048, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7cc163c917c94f87897dab982da693ba, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a3ad618400e74e318ff5016c903311a8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a0533f4dac9340348011268168e04f16] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=47.8 K 2024-11-09T06:18:04,311 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b90fe6d725b4cf5aaad058f6f773048, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731133077609 2024-11-09T06:18:04,311 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cc163c917c94f87897dab982da693ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731133079199 2024-11-09T06:18:04,311 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a3ad618400e74e318ff5016c903311a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731133079794 2024-11-09T06:18:04,312 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a0533f4dac9340348011268168e04f16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731133081389 2024-11-09T06:18:04,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b6a910450acb4084a03da664ff0a28a7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133083614/Put/seqid=0 2024-11-09T06:18:04,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133144312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:04,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133144316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:04,344 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#C#compaction#520 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:04,344 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/106d5c376d4349249d011c348540b749 is 50, key is test_row_0/C:col10/1731133083574/Put/seqid=0 2024-11-09T06:18:04,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742435_1611 (size=4469) 2024-11-09T06:18:04,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742436_1612 (size=14794) 2024-11-09T06:18:04,394 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:04,402 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b6a910450acb4084a03da664ff0a28a7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b6a910450acb4084a03da664ff0a28a7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:04,403 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b40d0ed444964f60a198429d69ba5f3e, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:04,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b40d0ed444964f60a198429d69ba5f3e is 175, key is test_row_0/A:col10/1731133083614/Put/seqid=0 2024-11-09T06:18:04,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742437_1613 (size=12595) 2024-11-09T06:18:04,427 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/106d5c376d4349249d011c348540b749 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/106d5c376d4349249d011c348540b749 2024-11-09T06:18:04,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133144433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:04,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133144433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:04,441 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 106d5c376d4349249d011c348540b749(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:04,441 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:04,441 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C, priority=12, startTime=1731133084231; duration=0sec 2024-11-09T06:18:04,441 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:04,441 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C 2024-11-09T06:18:04,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742438_1614 (size=39749) 2024-11-09T06:18:04,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133144643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:04,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:04,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133144643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T06:18:04,688 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:18:04,690 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:18:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-09T06:18:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T06:18:04,700 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:18:04,702 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:18:04,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:18:04,763 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#A#compaction#518 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:04,764 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/fe94e77786b442ca9cb353e3f9e70376 is 175, key is test_row_0/A:col10/1731133083574/Put/seqid=0 2024-11-09T06:18:04,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742439_1615 (size=31549) 2024-11-09T06:18:04,774 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/fe94e77786b442ca9cb353e3f9e70376 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/fe94e77786b442ca9cb353e3f9e70376 2024-11-09T06:18:04,781 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into fe94e77786b442ca9cb353e3f9e70376(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:04,781 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:04,781 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A, priority=12, startTime=1731133084228; duration=0sec 2024-11-09T06:18:04,781 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:04,781 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A 2024-11-09T06:18:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T06:18:04,851 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b40d0ed444964f60a198429d69ba5f3e 2024-11-09T06:18:04,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:04,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:04,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:04,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:04,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:04,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:04,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/c8d88991b4d14df5bdf9bed936d5438a is 50, key is test_row_0/B:col10/1731133083614/Put/seqid=0 2024-11-09T06:18:04,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742440_1616 (size=12151) 2024-11-09T06:18:04,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133144953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133144953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:05,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:05,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:05,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T06:18:05,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:05,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:05,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/c8d88991b4d14df5bdf9bed936d5438a 2024-11-09T06:18:05,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/a434d2b8d8e14609ba9e6db074123f2f is 50, key is test_row_0/C:col10/1731133083614/Put/seqid=0 2024-11-09T06:18:05,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742441_1617 (size=12151) 2024-11-09T06:18:05,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/a434d2b8d8e14609ba9e6db074123f2f 2024-11-09T06:18:05,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b40d0ed444964f60a198429d69ba5f3e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b40d0ed444964f60a198429d69ba5f3e 2024-11-09T06:18:05,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:05,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b40d0ed444964f60a198429d69ba5f3e, entries=200, sequenceid=230, filesize=38.8 K 2024-11-09T06:18:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T06:18:05,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/c8d88991b4d14df5bdf9bed936d5438a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c8d88991b4d14df5bdf9bed936d5438a 2024-11-09T06:18:05,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c8d88991b4d14df5bdf9bed936d5438a, entries=150, sequenceid=230, filesize=11.9 K 2024-11-09T06:18:05,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/a434d2b8d8e14609ba9e6db074123f2f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a434d2b8d8e14609ba9e6db074123f2f 2024-11-09T06:18:05,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a434d2b8d8e14609ba9e6db074123f2f, entries=150, sequenceid=230, filesize=11.9 K 2024-11-09T06:18:05,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 1077ms, sequenceid=230, compaction requested=false 2024-11-09T06:18:05,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:05,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T06:18:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:05,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:05,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:05,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096b504310eb5947bd95ec330a5dcecdca_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133084299/Put/seqid=0 2024-11-09T06:18:05,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742442_1618 (size=12304) 2024-11-09T06:18:05,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133145512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:05,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133145517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:05,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:05,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133145622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:05,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:05,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:05,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133145632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:05,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:05,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:05,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T06:18:05,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133145832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:05,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133145842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:05,911 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:05,919 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096b504310eb5947bd95ec330a5dcecdca_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096b504310eb5947bd95ec330a5dcecdca_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:05,920 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/6f8e28b001924dbcaced9aef2a429198, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:05,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/6f8e28b001924dbcaced9aef2a429198 is 175, key is test_row_0/A:col10/1731133084299/Put/seqid=0 2024-11-09T06:18:05,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:05,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:05,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:05,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:05,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742443_1619 (size=31105) 2024-11-09T06:18:06,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133146143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:06,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133146152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:06,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:06,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:06,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,350 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/6f8e28b001924dbcaced9aef2a429198 2024-11-09T06:18:06,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/0094cb5f9d484f66b2cadcf6d3e36d44 is 50, key is test_row_0/B:col10/1731133084299/Put/seqid=0 2024-11-09T06:18:06,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742444_1620 (size=12151) 2024-11-09T06:18:06,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:06,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:06,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:06,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:06,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133146653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:06,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133146662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:06,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:06,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:06,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T06:18:06,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/0094cb5f9d484f66b2cadcf6d3e36d44 2024-11-09T06:18:06,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/95e408dc5bc343f7ba6b126460d3b638 is 50, key is test_row_0/C:col10/1731133084299/Put/seqid=0 2024-11-09T06:18:06,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742445_1621 (size=12151) 2024-11-09T06:18:06,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/95e408dc5bc343f7ba6b126460d3b638 2024-11-09T06:18:06,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/6f8e28b001924dbcaced9aef2a429198 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6f8e28b001924dbcaced9aef2a429198 2024-11-09T06:18:06,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6f8e28b001924dbcaced9aef2a429198, entries=150, sequenceid=252, filesize=30.4 K 2024-11-09T06:18:06,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/0094cb5f9d484f66b2cadcf6d3e36d44 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/0094cb5f9d484f66b2cadcf6d3e36d44 2024-11-09T06:18:06,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/0094cb5f9d484f66b2cadcf6d3e36d44, entries=150, sequenceid=252, filesize=11.9 K 2024-11-09T06:18:06,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/95e408dc5bc343f7ba6b126460d3b638 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/95e408dc5bc343f7ba6b126460d3b638 2024-11-09T06:18:06,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/95e408dc5bc343f7ba6b126460d3b638, entries=150, sequenceid=252, filesize=11.9 K 2024-11-09T06:18:06,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 1368ms, sequenceid=252, compaction requested=true 2024-11-09T06:18:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:18:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:06,834 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:18:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:18:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:06,834 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:18:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:18:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:18:06,836 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:18:06,836 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B is initiating minor compaction (all files) 2024-11-09T06:18:06,836 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:18:06,836 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,836 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A is initiating minor compaction (all files) 2024-11-09T06:18:06,837 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/aee15b5ec6d54450891ba775c095ec1b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c8d88991b4d14df5bdf9bed936d5438a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/0094cb5f9d484f66b2cadcf6d3e36d44] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=36.0 K 2024-11-09T06:18:06,837 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,837 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/fe94e77786b442ca9cb353e3f9e70376, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b40d0ed444964f60a198429d69ba5f3e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6f8e28b001924dbcaced9aef2a429198] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=100.0 K 2024-11-09T06:18:06,837 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,837 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/fe94e77786b442ca9cb353e3f9e70376, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b40d0ed444964f60a198429d69ba5f3e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6f8e28b001924dbcaced9aef2a429198] 2024-11-09T06:18:06,837 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting aee15b5ec6d54450891ba775c095ec1b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731133081389 2024-11-09T06:18:06,837 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting fe94e77786b442ca9cb353e3f9e70376, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731133081389 2024-11-09T06:18:06,837 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8d88991b4d14df5bdf9bed936d5438a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731133083593 2024-11-09T06:18:06,838 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b40d0ed444964f60a198429d69ba5f3e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731133083593 2024-11-09T06:18:06,838 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f8e28b001924dbcaced9aef2a429198, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731133084299 2024-11-09T06:18:06,838 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0094cb5f9d484f66b2cadcf6d3e36d44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731133084299 2024-11-09T06:18:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T06:18:06,845 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#B#compaction#526 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:06,846 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/3891b077e74840f3ac4d151bca820ed0 is 50, key is test_row_0/B:col10/1731133084299/Put/seqid=0 2024-11-09T06:18:06,849 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:06,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T06:18:06,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,853 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:18:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:06,858 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411090815b52bacff48b69d89001837a4a70a_1aeacadfe7a3b9ebc8c8b0e6e39a10ca store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:06,860 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411090815b52bacff48b69d89001837a4a70a_1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:06,860 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090815b52bacff48b69d89001837a4a70a_1aeacadfe7a3b9ebc8c8b0e6e39a10ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:06,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742446_1622 (size=12697) 2024-11-09T06:18:06,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091d4ad233dfd849cbae34eb52f2cee875_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133085516/Put/seqid=0 2024-11-09T06:18:06,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742448_1624 (size=12454) 2024-11-09T06:18:06,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:06,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742447_1623 (size=4469) 2024-11-09T06:18:06,909 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091d4ad233dfd849cbae34eb52f2cee875_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091d4ad233dfd849cbae34eb52f2cee875_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:06,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/1cf6b74977a2443ebe8e920ac0714046, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:06,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/1cf6b74977a2443ebe8e920ac0714046 is 175, key is test_row_0/A:col10/1731133085516/Put/seqid=0 2024-11-09T06:18:06,912 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#A#compaction#527 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:06,913 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/f69de498cdb64bee8986c9c8d8ddbac8 is 175, key is test_row_0/A:col10/1731133084299/Put/seqid=0 2024-11-09T06:18:06,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742449_1625 (size=31255) 2024-11-09T06:18:06,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742450_1626 (size=31651) 2024-11-09T06:18:06,941 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/f69de498cdb64bee8986c9c8d8ddbac8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/f69de498cdb64bee8986c9c8d8ddbac8 2024-11-09T06:18:06,950 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into f69de498cdb64bee8986c9c8d8ddbac8(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:06,950 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:06,950 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A, priority=13, startTime=1731133086834; duration=0sec 2024-11-09T06:18:06,951 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:18:06,951 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A 2024-11-09T06:18:06,951 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:18:06,952 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:18:06,952 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C is initiating minor compaction (all files) 2024-11-09T06:18:06,952 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:06,953 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/106d5c376d4349249d011c348540b749, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a434d2b8d8e14609ba9e6db074123f2f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/95e408dc5bc343f7ba6b126460d3b638] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=36.0 K 2024-11-09T06:18:06,953 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 106d5c376d4349249d011c348540b749, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731133081389 2024-11-09T06:18:06,953 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a434d2b8d8e14609ba9e6db074123f2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731133083593 2024-11-09T06:18:06,954 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e408dc5bc343f7ba6b126460d3b638, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731133084299 2024-11-09T06:18:06,966 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#C#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:06,966 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/8829207263bd4411ba4faf158259740b is 50, key is test_row_0/C:col10/1731133084299/Put/seqid=0 2024-11-09T06:18:06,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742451_1627 (size=12697) 2024-11-09T06:18:07,276 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/3891b077e74840f3ac4d151bca820ed0 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3891b077e74840f3ac4d151bca820ed0 2024-11-09T06:18:07,286 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 3891b077e74840f3ac4d151bca820ed0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:07,286 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:07,287 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B, priority=13, startTime=1731133086834; duration=0sec 2024-11-09T06:18:07,287 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:07,287 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B 2024-11-09T06:18:07,328 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/1cf6b74977a2443ebe8e920ac0714046 2024-11-09T06:18:07,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/a4dad32abd884377bde9ef018f347785 is 50, key is test_row_0/B:col10/1731133085516/Put/seqid=0 2024-11-09T06:18:07,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742452_1628 (size=12301) 2024-11-09T06:18:07,362 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/a4dad32abd884377bde9ef018f347785 2024-11-09T06:18:07,379 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/8829207263bd4411ba4faf158259740b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/8829207263bd4411ba4faf158259740b 2024-11-09T06:18:07,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/b0a6eb0d0e2c4413b19ce675922bbb3a is 50, key is test_row_0/C:col10/1731133085516/Put/seqid=0 2024-11-09T06:18:07,384 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 8829207263bd4411ba4faf158259740b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:07,384 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:07,384 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C, priority=13, startTime=1731133086834; duration=0sec 2024-11-09T06:18:07,384 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:07,385 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C 2024-11-09T06:18:07,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742453_1629 (size=12301) 2024-11-09T06:18:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:07,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133147713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:07,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133147718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:07,811 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/b0a6eb0d0e2c4413b19ce675922bbb3a 2024-11-09T06:18:07,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/1cf6b74977a2443ebe8e920ac0714046 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1cf6b74977a2443ebe8e920ac0714046 2024-11-09T06:18:07,819 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1cf6b74977a2443ebe8e920ac0714046, entries=150, sequenceid=265, filesize=30.5 K 2024-11-09T06:18:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/a4dad32abd884377bde9ef018f347785 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a4dad32abd884377bde9ef018f347785 2024-11-09T06:18:07,823 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a4dad32abd884377bde9ef018f347785, entries=150, sequenceid=265, filesize=12.0 K 2024-11-09T06:18:07,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/b0a6eb0d0e2c4413b19ce675922bbb3a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b0a6eb0d0e2c4413b19ce675922bbb3a 2024-11-09T06:18:07,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133147823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:07,829 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b0a6eb0d0e2c4413b19ce675922bbb3a, entries=150, sequenceid=265, filesize=12.0 K 2024-11-09T06:18:07,830 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 977ms, sequenceid=265, compaction requested=false 2024-11-09T06:18:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-09T06:18:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-09T06:18:07,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-09T06:18:07,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1290 sec 2024-11-09T06:18:07,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.1430 sec 2024-11-09T06:18:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:07,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T06:18:07,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:07,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:07,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:07,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:07,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:07,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:07,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092728344da9b8474693258d272d03c27d_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133087716/Put/seqid=0 2024-11-09T06:18:07,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742454_1630 (size=14994) 2024-11-09T06:18:07,853 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:07,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092728344da9b8474693258d272d03c27d_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092728344da9b8474693258d272d03c27d_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:07,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/5b8ff8bd65a04cc68b6023641bacefda, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:07,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/5b8ff8bd65a04cc68b6023641bacefda is 175, key is test_row_0/A:col10/1731133087716/Put/seqid=0 2024-11-09T06:18:07,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742455_1631 (size=39949) 2024-11-09T06:18:07,873 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/5b8ff8bd65a04cc68b6023641bacefda 2024-11-09T06:18:07,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/b46ed9aa57f348089ecc4a92535a7421 is 50, key is test_row_0/B:col10/1731133087716/Put/seqid=0 2024-11-09T06:18:07,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:07,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133147882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:07,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742456_1632 (size=12301) 2024-11-09T06:18:07,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/b46ed9aa57f348089ecc4a92535a7421 2024-11-09T06:18:07,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/cc19a363f12e4bd59a362abb8b1e12e4 is 50, key is test_row_0/C:col10/1731133087716/Put/seqid=0 2024-11-09T06:18:07,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742457_1633 (size=12301) 2024-11-09T06:18:07,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/cc19a363f12e4bd59a362abb8b1e12e4 2024-11-09T06:18:07,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/5b8ff8bd65a04cc68b6023641bacefda as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5b8ff8bd65a04cc68b6023641bacefda 2024-11-09T06:18:07,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5b8ff8bd65a04cc68b6023641bacefda, entries=200, sequenceid=288, filesize=39.0 K 2024-11-09T06:18:07,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/b46ed9aa57f348089ecc4a92535a7421 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/b46ed9aa57f348089ecc4a92535a7421 2024-11-09T06:18:07,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/b46ed9aa57f348089ecc4a92535a7421, entries=150, sequenceid=288, filesize=12.0 K 2024-11-09T06:18:07,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/cc19a363f12e4bd59a362abb8b1e12e4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/cc19a363f12e4bd59a362abb8b1e12e4 2024-11-09T06:18:07,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/cc19a363f12e4bd59a362abb8b1e12e4, entries=150, sequenceid=288, filesize=12.0 K 2024-11-09T06:18:07,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 121ms, sequenceid=288, compaction requested=true 2024-11-09T06:18:07,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:07,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:18:07,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:07,958 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:18:07,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:18:07,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:07,958 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:18:07,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:18:07,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:18:07,959 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:18:07,959 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:18:07,959 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A is initiating minor compaction (all files) 2024-11-09T06:18:07,959 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B is initiating minor compaction (all files) 2024-11-09T06:18:07,959 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:07,959 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:07,959 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3891b077e74840f3ac4d151bca820ed0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a4dad32abd884377bde9ef018f347785, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/b46ed9aa57f348089ecc4a92535a7421] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=36.4 K 2024-11-09T06:18:07,959 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/f69de498cdb64bee8986c9c8d8ddbac8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1cf6b74977a2443ebe8e920ac0714046, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5b8ff8bd65a04cc68b6023641bacefda] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=100.4 K 2024-11-09T06:18:07,959 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3891b077e74840f3ac4d151bca820ed0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731133084299 2024-11-09T06:18:07,959 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:07,959 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/f69de498cdb64bee8986c9c8d8ddbac8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1cf6b74977a2443ebe8e920ac0714046, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5b8ff8bd65a04cc68b6023641bacefda] 2024-11-09T06:18:07,960 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f69de498cdb64bee8986c9c8d8ddbac8, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731133084299 2024-11-09T06:18:07,960 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cf6b74977a2443ebe8e920ac0714046, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731133085471 2024-11-09T06:18:07,961 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a4dad32abd884377bde9ef018f347785, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731133085471 2024-11-09T06:18:07,961 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b8ff8bd65a04cc68b6023641bacefda, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731133087703 2024-11-09T06:18:07,961 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b46ed9aa57f348089ecc4a92535a7421, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731133087710 2024-11-09T06:18:07,979 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#B#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:07,979 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/506d5fda1471434d9b445a1ded801177 is 50, key is test_row_0/B:col10/1731133087716/Put/seqid=0 2024-11-09T06:18:07,989 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:07,996 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110960b29b1baff9445c8c9f086f3b0c9d19_1aeacadfe7a3b9ebc8c8b0e6e39a10ca store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:07,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T06:18:07,998 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110960b29b1baff9445c8c9f086f3b0c9d19_1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:07,998 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110960b29b1baff9445c8c9f086f3b0c9d19_1aeacadfe7a3b9ebc8c8b0e6e39a10ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:07,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:07,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:07,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:07,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:07,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:08,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:08,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742458_1634 (size=12949) 2024-11-09T06:18:08,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110968dc907c74e6407ea6e65f9f08c7086a_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133087995/Put/seqid=0 2024-11-09T06:18:08,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133148032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133148034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133148033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45946 deadline: 1731133148034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,039 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:18:08,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133148039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742459_1635 (size=4469) 2024-11-09T06:18:08,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742460_1636 (size=12454) 2024-11-09T06:18:08,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:08,069 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110968dc907c74e6407ea6e65f9f08c7086a_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110968dc907c74e6407ea6e65f9f08c7086a_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:08,070 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/09d20d5284004764b1d92935ded3b316, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:08,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/09d20d5284004764b1d92935ded3b316 is 175, key is test_row_0/A:col10/1731133087995/Put/seqid=0 2024-11-09T06:18:08,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742461_1637 (size=31255) 2024-11-09T06:18:08,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133148143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133148143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133148152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133148342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133148353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133148353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133148362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,410 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/506d5fda1471434d9b445a1ded801177 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/506d5fda1471434d9b445a1ded801177 2024-11-09T06:18:08,415 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 506d5fda1471434d9b445a1ded801177(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:08,415 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:08,415 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B, priority=13, startTime=1731133087958; duration=0sec 2024-11-09T06:18:08,415 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:18:08,415 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B 2024-11-09T06:18:08,415 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:18:08,417 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:18:08,417 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C is initiating minor compaction (all files) 2024-11-09T06:18:08,417 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:08,417 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/8829207263bd4411ba4faf158259740b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b0a6eb0d0e2c4413b19ce675922bbb3a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/cc19a363f12e4bd59a362abb8b1e12e4] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=36.4 K 2024-11-09T06:18:08,417 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8829207263bd4411ba4faf158259740b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731133084299 2024-11-09T06:18:08,418 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b0a6eb0d0e2c4413b19ce675922bbb3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731133085471 2024-11-09T06:18:08,418 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting cc19a363f12e4bd59a362abb8b1e12e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731133087710 2024-11-09T06:18:08,426 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#C#compaction#538 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:08,427 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/9f39759b21c446f99b0399928c3049a4 is 50, key is test_row_0/C:col10/1731133087716/Put/seqid=0 2024-11-09T06:18:08,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742462_1638 (size=12949) 2024-11-09T06:18:08,456 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#A#compaction#536 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:08,457 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/cb286d38c7ad43aca8c7fa443dbf46e7 is 175, key is test_row_0/A:col10/1731133087716/Put/seqid=0 2024-11-09T06:18:08,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742463_1639 (size=31903) 2024-11-09T06:18:08,470 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/cb286d38c7ad43aca8c7fa443dbf46e7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cb286d38c7ad43aca8c7fa443dbf46e7 2024-11-09T06:18:08,477 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/09d20d5284004764b1d92935ded3b316 2024-11-09T06:18:08,477 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into cb286d38c7ad43aca8c7fa443dbf46e7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:08,477 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:08,478 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A, priority=13, startTime=1731133087958; duration=0sec 2024-11-09T06:18:08,478 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:08,478 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A 2024-11-09T06:18:08,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/a18733bc4c9b424da6b9f2ad91fd0beb is 50, key is test_row_0/B:col10/1731133087995/Put/seqid=0 2024-11-09T06:18:08,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742464_1640 (size=12301) 2024-11-09T06:18:08,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133148663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133148663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133148672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,850 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/9f39759b21c446f99b0399928c3049a4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/9f39759b21c446f99b0399928c3049a4 2024-11-09T06:18:08,854 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 9f39759b21c446f99b0399928c3049a4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:08,854 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:08,854 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C, priority=13, startTime=1731133087958; duration=0sec 2024-11-09T06:18:08,855 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:08,855 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C 2024-11-09T06:18:08,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133148853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T06:18:08,863 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:18:08,863 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:18:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-09T06:18:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T06:18:08,866 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:18:08,866 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:18:08,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:18:08,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/a18733bc4c9b424da6b9f2ad91fd0beb 2024-11-09T06:18:08,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/c4bea2b2583745dcb4d55c04cbed95ce is 50, key is test_row_0/C:col10/1731133087995/Put/seqid=0 2024-11-09T06:18:08,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742465_1641 (size=12301) 2024-11-09T06:18:08,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/c4bea2b2583745dcb4d55c04cbed95ce 2024-11-09T06:18:08,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/09d20d5284004764b1d92935ded3b316 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/09d20d5284004764b1d92935ded3b316 2024-11-09T06:18:08,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/09d20d5284004764b1d92935ded3b316, entries=150, sequenceid=302, filesize=30.5 K 2024-11-09T06:18:08,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/a18733bc4c9b424da6b9f2ad91fd0beb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a18733bc4c9b424da6b9f2ad91fd0beb 2024-11-09T06:18:08,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a18733bc4c9b424da6b9f2ad91fd0beb, entries=150, sequenceid=302, filesize=12.0 K 2024-11-09T06:18:08,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/c4bea2b2583745dcb4d55c04cbed95ce as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c4bea2b2583745dcb4d55c04cbed95ce 2024-11-09T06:18:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T06:18:08,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c4bea2b2583745dcb4d55c04cbed95ce, entries=150, sequenceid=302, filesize=12.0 K 2024-11-09T06:18:08,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 981ms, sequenceid=302, compaction requested=false 2024-11-09T06:18:08,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:09,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T06:18:09,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:09,019 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T06:18:09,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:09,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:09,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:09,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:09,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:09,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:09,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110976d3c6b18081445ca2cc19a5c5a0e3f2_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133088030/Put/seqid=0 2024-11-09T06:18:09,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742466_1642 (size=12454) 2024-11-09T06:18:09,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:09,051 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110976d3c6b18081445ca2cc19a5c5a0e3f2_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110976d3c6b18081445ca2cc19a5c5a0e3f2_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/557c424be65346078b29da24d7dc85cc, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/557c424be65346078b29da24d7dc85cc is 175, key is test_row_0/A:col10/1731133088030/Put/seqid=0 2024-11-09T06:18:09,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742467_1643 (size=31255) 2024-11-09T06:18:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:09,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T06:18:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133149195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133149200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133149201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133149302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133149312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133149313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,474 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/557c424be65346078b29da24d7dc85cc 2024-11-09T06:18:09,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/5dd6405de56d40c9937a4509ef4599c8 is 50, key is test_row_0/B:col10/1731133088030/Put/seqid=0 2024-11-09T06:18:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T06:18:09,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742468_1644 (size=12301) 2024-11-09T06:18:09,506 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/5dd6405de56d40c9937a4509ef4599c8 2024-11-09T06:18:09,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/5378cc909b0c4bcd8507af73f6edb010 is 50, key is test_row_0/C:col10/1731133088030/Put/seqid=0 2024-11-09T06:18:09,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133149513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133149522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133149522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742469_1645 (size=12301) 2024-11-09T06:18:09,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133149823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133149833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133149833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133149872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:09,877 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a84bcce7bad,46317,1731132929173 for 'test_row_2' in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T06:18:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:09,935 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/5378cc909b0c4bcd8507af73f6edb010 2024-11-09T06:18:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/557c424be65346078b29da24d7dc85cc as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/557c424be65346078b29da24d7dc85cc 2024-11-09T06:18:09,943 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/557c424be65346078b29da24d7dc85cc, entries=150, sequenceid=323, filesize=30.5 K 2024-11-09T06:18:09,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/5dd6405de56d40c9937a4509ef4599c8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5dd6405de56d40c9937a4509ef4599c8 2024-11-09T06:18:09,947 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5dd6405de56d40c9937a4509ef4599c8, entries=150, sequenceid=323, filesize=12.0 K 2024-11-09T06:18:09,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/5378cc909b0c4bcd8507af73f6edb010 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5378cc909b0c4bcd8507af73f6edb010 2024-11-09T06:18:09,959 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5378cc909b0c4bcd8507af73f6edb010, entries=150, sequenceid=323, filesize=12.0 K 2024-11-09T06:18:09,960 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 941ms, sequenceid=323, compaction requested=true 2024-11-09T06:18:09,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:09,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:09,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-09T06:18:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-09T06:18:09,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-09T06:18:09,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0950 sec 2024-11-09T06:18:09,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.1000 sec 2024-11-09T06:18:10,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T06:18:10,003 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:18:10,003 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:18:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-09T06:18:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T06:18:10,005 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:18:10,007 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:18:10,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:18:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T06:18:10,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T06:18:10,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:10,162 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T06:18:10,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:10,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:10,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:10,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109204a8efe82cf40ea80d82933b41e2eef_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133089198/Put/seqid=0 2024-11-09T06:18:10,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742470_1646 (size=12454) 2024-11-09T06:18:10,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:10,195 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109204a8efe82cf40ea80d82933b41e2eef_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109204a8efe82cf40ea80d82933b41e2eef_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:10,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/4c6b45da6341406481ebbbdc96ecfe27, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:10,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/4c6b45da6341406481ebbbdc96ecfe27 is 175, key is test_row_0/A:col10/1731133089198/Put/seqid=0 2024-11-09T06:18:10,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742471_1647 (size=31255) 2024-11-09T06:18:10,208 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/4c6b45da6341406481ebbbdc96ecfe27 2024-11-09T06:18:10,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/c33b8b4f5c054cf59911e662d026293a is 50, key is test_row_0/B:col10/1731133089198/Put/seqid=0 2024-11-09T06:18:10,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742472_1648 (size=12301) 2024-11-09T06:18:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T06:18:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:10,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:10,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133150370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:10,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133150372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:10,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133150377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:10,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133150482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:10,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133150482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:10,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133150493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T06:18:10,645 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/c33b8b4f5c054cf59911e662d026293a 2024-11-09T06:18:10,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/77bee433a9b2461cb9d5046c32aeacec is 50, key is test_row_0/C:col10/1731133089198/Put/seqid=0 2024-11-09T06:18:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742473_1649 (size=12301) 2024-11-09T06:18:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133150693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133150693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:10,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133150703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133151003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133151003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133151020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,070 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/77bee433a9b2461cb9d5046c32aeacec 2024-11-09T06:18:11,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/4c6b45da6341406481ebbbdc96ecfe27 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4c6b45da6341406481ebbbdc96ecfe27 2024-11-09T06:18:11,077 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4c6b45da6341406481ebbbdc96ecfe27, entries=150, sequenceid=338, filesize=30.5 K 2024-11-09T06:18:11,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/c33b8b4f5c054cf59911e662d026293a as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c33b8b4f5c054cf59911e662d026293a 2024-11-09T06:18:11,084 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c33b8b4f5c054cf59911e662d026293a, entries=150, sequenceid=338, filesize=12.0 K 2024-11-09T06:18:11,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/77bee433a9b2461cb9d5046c32aeacec as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/77bee433a9b2461cb9d5046c32aeacec 2024-11-09T06:18:11,091 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/77bee433a9b2461cb9d5046c32aeacec, entries=150, sequenceid=338, filesize=12.0 K 2024-11-09T06:18:11,092 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 930ms, sequenceid=338, compaction requested=true 2024-11-09T06:18:11,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:11,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:11,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-09T06:18:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-09T06:18:11,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-09T06:18:11,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0880 sec 2024-11-09T06:18:11,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.0950 sec 2024-11-09T06:18:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T06:18:11,143 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:18:11,144 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:18:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-09T06:18:11,146 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:18:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T06:18:11,147 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:18:11,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:18:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T06:18:11,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T06:18:11,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:11,301 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T06:18:11,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:11,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:11,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:11,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:11,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:11,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:11,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ba53a310437947aabe9c42acdfef4507_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133090371/Put/seqid=0 2024-11-09T06:18:11,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742474_1650 (size=12454) 2024-11-09T06:18:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T06:18:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:11,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:11,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133151540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133151543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133151544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133151652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133151652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133151652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:11,731 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ba53a310437947aabe9c42acdfef4507_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ba53a310437947aabe9c42acdfef4507_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:11,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/8d9b4a0876df4e70ba805c91df7d5445, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:11,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/8d9b4a0876df4e70ba805c91df7d5445 is 175, key is test_row_0/A:col10/1731133090371/Put/seqid=0 2024-11-09T06:18:11,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742475_1651 (size=31255) 2024-11-09T06:18:11,739 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/8d9b4a0876df4e70ba805c91df7d5445 2024-11-09T06:18:11,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/2294adb4056f4d019a56a38b6fb08b4e is 50, key is test_row_0/B:col10/1731133090371/Put/seqid=0 2024-11-09T06:18:11,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742476_1652 (size=12301) 2024-11-09T06:18:11,759 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/2294adb4056f4d019a56a38b6fb08b4e 2024-11-09T06:18:11,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/612a35f6c7b74e21837cc675371788b5 is 50, key is test_row_0/C:col10/1731133090371/Put/seqid=0 2024-11-09T06:18:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T06:18:11,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742477_1653 (size=12301) 2024-11-09T06:18:11,808 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/612a35f6c7b74e21837cc675371788b5 2024-11-09T06:18:11,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/8d9b4a0876df4e70ba805c91df7d5445 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8d9b4a0876df4e70ba805c91df7d5445 2024-11-09T06:18:11,817 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8d9b4a0876df4e70ba805c91df7d5445, entries=150, sequenceid=355, filesize=30.5 K 2024-11-09T06:18:11,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/2294adb4056f4d019a56a38b6fb08b4e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/2294adb4056f4d019a56a38b6fb08b4e 2024-11-09T06:18:11,821 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/2294adb4056f4d019a56a38b6fb08b4e, entries=150, sequenceid=355, filesize=12.0 K 2024-11-09T06:18:11,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/612a35f6c7b74e21837cc675371788b5 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/612a35f6c7b74e21837cc675371788b5 2024-11-09T06:18:11,825 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/612a35f6c7b74e21837cc675371788b5, entries=150, sequenceid=355, filesize=12.0 K 2024-11-09T06:18:11,826 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 525ms, sequenceid=355, compaction requested=true 2024-11-09T06:18:11,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:11,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:11,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-09T06:18:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-09T06:18:11,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-09T06:18:11,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 680 msec 2024-11-09T06:18:11,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 686 msec 2024-11-09T06:18:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:11,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T06:18:11,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:11,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:11,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:11,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:11,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:11,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:11,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d555c627e0ec4acea9be82e87fdd6c53_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133091865/Put/seqid=0 2024-11-09T06:18:11,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742478_1654 (size=14994) 2024-11-09T06:18:11,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133151889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133151890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133151891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:11,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133151894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:11,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:11,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133152003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133152004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133152004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133152004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133152212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133152212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133152212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133152212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T06:18:12,283 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:18:12,284 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T06:18:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-09T06:18:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T06:18:12,286 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T06:18:12,286 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:12,287 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T06:18:12,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T06:18:12,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d555c627e0ec4acea9be82e87fdd6c53_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d555c627e0ec4acea9be82e87fdd6c53_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:12,294 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b5f66a422c67400891fdcf8a83a8997f, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:12,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b5f66a422c67400891fdcf8a83a8997f is 175, key is test_row_0/A:col10/1731133091865/Put/seqid=0 2024-11-09T06:18:12,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742479_1655 (size=39949) 2024-11-09T06:18:12,303 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b5f66a422c67400891fdcf8a83a8997f 2024-11-09T06:18:12,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/9963025bde7547788a586d49cddda777 is 50, key is test_row_0/B:col10/1731133091865/Put/seqid=0 2024-11-09T06:18:12,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742480_1656 (size=12301) 2024-11-09T06:18:12,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/9963025bde7547788a586d49cddda777 2024-11-09T06:18:12,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/e50811313c6b499d959baef9602d6f96 is 50, key is test_row_0/C:col10/1731133091865/Put/seqid=0 2024-11-09T06:18:12,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742481_1657 (size=12301) 2024-11-09T06:18:12,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/e50811313c6b499d959baef9602d6f96 2024-11-09T06:18:12,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b5f66a422c67400891fdcf8a83a8997f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5f66a422c67400891fdcf8a83a8997f 2024-11-09T06:18:12,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5f66a422c67400891fdcf8a83a8997f, entries=200, sequenceid=371, filesize=39.0 K 2024-11-09T06:18:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T06:18:12,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/9963025bde7547788a586d49cddda777 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/9963025bde7547788a586d49cddda777 2024-11-09T06:18:12,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/9963025bde7547788a586d49cddda777, entries=150, sequenceid=371, filesize=12.0 K 2024-11-09T06:18:12,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/e50811313c6b499d959baef9602d6f96 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/e50811313c6b499d959baef9602d6f96 2024-11-09T06:18:12,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/e50811313c6b499d959baef9602d6f96, entries=150, sequenceid=371, filesize=12.0 K 2024-11-09T06:18:12,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 539ms, sequenceid=371, compaction requested=true 2024-11-09T06:18:12,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:12,405 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T06:18:12,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:18:12,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:12,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:18:12,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:12,406 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T06:18:12,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:18:12,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:18:12,407 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 196872 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T06:18:12,407 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A is initiating minor compaction (all files) 2024-11-09T06:18:12,407 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:12,407 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cb286d38c7ad43aca8c7fa443dbf46e7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/09d20d5284004764b1d92935ded3b316, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/557c424be65346078b29da24d7dc85cc, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4c6b45da6341406481ebbbdc96ecfe27, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8d9b4a0876df4e70ba805c91df7d5445, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5f66a422c67400891fdcf8a83a8997f] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=192.3 K 2024-11-09T06:18:12,407 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:12,407 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cb286d38c7ad43aca8c7fa443dbf46e7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/09d20d5284004764b1d92935ded3b316, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/557c424be65346078b29da24d7dc85cc, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4c6b45da6341406481ebbbdc96ecfe27, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8d9b4a0876df4e70ba805c91df7d5445, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5f66a422c67400891fdcf8a83a8997f] 2024-11-09T06:18:12,407 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb286d38c7ad43aca8c7fa443dbf46e7, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731133087710 2024-11-09T06:18:12,407 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74454 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T06:18:12,407 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B is initiating minor compaction (all files) 2024-11-09T06:18:12,407 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09d20d5284004764b1d92935ded3b316, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731133087862 2024-11-09T06:18:12,407 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:12,408 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/506d5fda1471434d9b445a1ded801177, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a18733bc4c9b424da6b9f2ad91fd0beb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5dd6405de56d40c9937a4509ef4599c8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c33b8b4f5c054cf59911e662d026293a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/2294adb4056f4d019a56a38b6fb08b4e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/9963025bde7547788a586d49cddda777] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=72.7 K 2024-11-09T06:18:12,408 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 557c424be65346078b29da24d7dc85cc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731133088030 2024-11-09T06:18:12,408 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c6b45da6341406481ebbbdc96ecfe27, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731133089192 2024-11-09T06:18:12,408 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 506d5fda1471434d9b445a1ded801177, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731133087710 2024-11-09T06:18:12,408 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting a18733bc4c9b424da6b9f2ad91fd0beb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731133087862 2024-11-09T06:18:12,408 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d9b4a0876df4e70ba805c91df7d5445, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731133090360 2024-11-09T06:18:12,409 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5f66a422c67400891fdcf8a83a8997f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731133091530 2024-11-09T06:18:12,409 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dd6405de56d40c9937a4509ef4599c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731133088030 2024-11-09T06:18:12,409 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting c33b8b4f5c054cf59911e662d026293a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731133089192 2024-11-09T06:18:12,410 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2294adb4056f4d019a56a38b6fb08b4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731133090360 2024-11-09T06:18:12,410 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9963025bde7547788a586d49cddda777, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731133091530 2024-11-09T06:18:12,422 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:12,425 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110916085e8cec26407db016eb3223f72ff5_1aeacadfe7a3b9ebc8c8b0e6e39a10ca store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:12,430 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110916085e8cec26407db016eb3223f72ff5_1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:12,430 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110916085e8cec26407db016eb3223f72ff5_1aeacadfe7a3b9ebc8c8b0e6e39a10ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:12,436 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#B#compaction#554 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:12,436 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/3246a628ce7242e5bbf3ab891a32fb66 is 50, key is test_row_0/B:col10/1731133091865/Put/seqid=0 2024-11-09T06:18:12,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T06:18:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:12,443 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T06:18:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:12,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742482_1658 (size=4469) 2024-11-09T06:18:12,472 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#A#compaction#553 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:12,473 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/6b21ddc5765e4396947e3464755a5773 is 175, key is test_row_0/A:col10/1731133091865/Put/seqid=0 2024-11-09T06:18:12,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090acdf77598454e5ab3729c7d08e6ad39_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133091878/Put/seqid=0 2024-11-09T06:18:12,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742483_1659 (size=13153) 2024-11-09T06:18:12,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742484_1660 (size=32107) 2024-11-09T06:18:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:12,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. as already flushing 2024-11-09T06:18:12,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742485_1661 (size=12454) 2024-11-09T06:18:12,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133152575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133152576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133152578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133152581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T06:18:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133152693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133152693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133152693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133152694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133152902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133152902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133152902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133152903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:12,909 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/3246a628ce7242e5bbf3ab891a32fb66 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3246a628ce7242e5bbf3ab891a32fb66 2024-11-09T06:18:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T06:18:12,913 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 3246a628ce7242e5bbf3ab891a32fb66(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:12,913 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:12,913 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B, priority=10, startTime=1731133092406; duration=0sec 2024-11-09T06:18:12,913 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:18:12,913 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B 2024-11-09T06:18:12,913 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T06:18:12,915 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74454 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T06:18:12,915 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C is initiating minor compaction (all files) 2024-11-09T06:18:12,915 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:12,915 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/9f39759b21c446f99b0399928c3049a4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c4bea2b2583745dcb4d55c04cbed95ce, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5378cc909b0c4bcd8507af73f6edb010, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/77bee433a9b2461cb9d5046c32aeacec, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/612a35f6c7b74e21837cc675371788b5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/e50811313c6b499d959baef9602d6f96] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=72.7 K 2024-11-09T06:18:12,915 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f39759b21c446f99b0399928c3049a4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731133087710 2024-11-09T06:18:12,915 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting c4bea2b2583745dcb4d55c04cbed95ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731133087862 2024-11-09T06:18:12,915 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5378cc909b0c4bcd8507af73f6edb010, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731133088030 2024-11-09T06:18:12,916 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 77bee433a9b2461cb9d5046c32aeacec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731133089192 2024-11-09T06:18:12,916 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 612a35f6c7b74e21837cc675371788b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731133090360 2024-11-09T06:18:12,916 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting e50811313c6b499d959baef9602d6f96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731133091530 2024-11-09T06:18:12,923 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/6b21ddc5765e4396947e3464755a5773 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6b21ddc5765e4396947e3464755a5773 2024-11-09T06:18:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:12,935 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 6b21ddc5765e4396947e3464755a5773(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:12,935 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:12,935 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A, priority=10, startTime=1731133092405; duration=0sec 2024-11-09T06:18:12,935 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:12,936 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A 2024-11-09T06:18:12,938 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090acdf77598454e5ab3729c7d08e6ad39_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090acdf77598454e5ab3729c7d08e6ad39_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:12,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/580415f2cc6a4d6ab6a49d86c050b903, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:12,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/580415f2cc6a4d6ab6a49d86c050b903 is 175, key is test_row_0/A:col10/1731133091878/Put/seqid=0 2024-11-09T06:18:12,941 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#C#compaction#556 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:12,942 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/b1c84dc61ac34413a3ed59fa31df5f5f is 50, key is test_row_0/C:col10/1731133091865/Put/seqid=0 2024-11-09T06:18:12,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742486_1662 (size=31255) 2024-11-09T06:18:12,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742487_1663 (size=13153) 2024-11-09T06:18:12,982 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/580415f2cc6a4d6ab6a49d86c050b903 2024-11-09T06:18:12,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/ca382e07bf5f44bca6e7ba8ae4569a40 is 50, key is test_row_0/B:col10/1731133091878/Put/seqid=0 2024-11-09T06:18:13,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742488_1664 (size=12301) 2024-11-09T06:18:13,010 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/ca382e07bf5f44bca6e7ba8ae4569a40 2024-11-09T06:18:13,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/fd210f7915e84e369568450f5ca32f4e is 50, key is test_row_0/C:col10/1731133091878/Put/seqid=0 2024-11-09T06:18:13,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742489_1665 (size=12301) 2024-11-09T06:18:13,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133153212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133153212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133153212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:13,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133153222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:13,387 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/b1c84dc61ac34413a3ed59fa31df5f5f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b1c84dc61ac34413a3ed59fa31df5f5f 2024-11-09T06:18:13,395 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into b1c84dc61ac34413a3ed59fa31df5f5f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:13,395 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:13,395 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C, priority=10, startTime=1731133092406; duration=0sec 2024-11-09T06:18:13,396 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:13,396 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C 2024-11-09T06:18:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T06:18:13,443 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/fd210f7915e84e369568450f5ca32f4e 2024-11-09T06:18:13,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/580415f2cc6a4d6ab6a49d86c050b903 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/580415f2cc6a4d6ab6a49d86c050b903 2024-11-09T06:18:13,452 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/580415f2cc6a4d6ab6a49d86c050b903, entries=150, sequenceid=387, filesize=30.5 K 2024-11-09T06:18:13,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/ca382e07bf5f44bca6e7ba8ae4569a40 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/ca382e07bf5f44bca6e7ba8ae4569a40 2024-11-09T06:18:13,457 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/ca382e07bf5f44bca6e7ba8ae4569a40, entries=150, sequenceid=387, filesize=12.0 K 2024-11-09T06:18:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/fd210f7915e84e369568450f5ca32f4e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fd210f7915e84e369568450f5ca32f4e 2024-11-09T06:18:13,461 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fd210f7915e84e369568450f5ca32f4e, entries=150, sequenceid=387, filesize=12.0 K 2024-11-09T06:18:13,462 INFO [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 1019ms, sequenceid=387, compaction requested=false 2024-11-09T06:18:13,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:13,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:13,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a84bcce7bad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-09T06:18:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-09T06:18:13,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-09T06:18:13,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1760 sec 2024-11-09T06:18:13,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.1820 sec 2024-11-09T06:18:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:13,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T06:18:13,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:13,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:13,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:13,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:13,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:13,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:13,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110987d56f195f7044a98e387a2e2ae8b370_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133092576/Put/seqid=0 2024-11-09T06:18:13,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133153739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:13,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133153743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:13,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133153745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:13,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133153747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:13,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742490_1666 (size=17534) 2024-11-09T06:18:13,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133153853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:13,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133153853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133153854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133153863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:14,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:14,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133154063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133154063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:14,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:14,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133154063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:14,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:14,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133154072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:14,155 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:14,159 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110987d56f195f7044a98e387a2e2ae8b370_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110987d56f195f7044a98e387a2e2ae8b370_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:14,160 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b5593b66d528481a9e585c89e028f581, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:14,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b5593b66d528481a9e585c89e028f581 is 175, key is test_row_0/A:col10/1731133092576/Put/seqid=0 2024-11-09T06:18:14,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742491_1667 (size=48639) 2024-11-09T06:18:14,187 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b5593b66d528481a9e585c89e028f581 2024-11-09T06:18:14,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/79b8abdfd403498681fc61d87f4e2739 is 50, key is test_row_0/B:col10/1731133092576/Put/seqid=0 2024-11-09T06:18:14,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742492_1668 (size=12301) 2024-11-09T06:18:14,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/79b8abdfd403498681fc61d87f4e2739 2024-11-09T06:18:14,219 INFO [Thread-2490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2490. 2024-11-09T06:18:14,219 DEBUG [Thread-2490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:18:14,219 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,219 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,219 INFO [Thread-2484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2484. 2024-11-09T06:18:14,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:14,219 DEBUG [Thread-2484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:18:14,219 DEBUG [Thread-2484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,219 DEBUG [Thread-2484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:14,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/63aee012f706443cb4715580bc067af7 is 50, key is test_row_0/C:col10/1731133092576/Put/seqid=0 2024-11-09T06:18:14,222 INFO [Thread-2488 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2488. 2024-11-09T06:18:14,222 DEBUG [Thread-2488 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:18:14,222 DEBUG [Thread-2488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,222 INFO [Thread-2486 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2486. 2024-11-09T06:18:14,222 DEBUG [Thread-2488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,222 DEBUG [Thread-2486 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:18:14,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:14,222 DEBUG [Thread-2486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:14,223 DEBUG [Thread-2486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,224 INFO [Thread-2482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2482. 2024-11-09T06:18:14,224 DEBUG [Thread-2482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:18:14,224 DEBUG [Thread-2482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,224 DEBUG [Thread-2482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:14,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742493_1669 (size=12301) 2024-11-09T06:18:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45956 deadline: 1731133154373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45924 deadline: 1731133154373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45920 deadline: 1731133154373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:14,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T06:18:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46317 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45900 deadline: 1731133154382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T06:18:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., hostname=4a84bcce7bad,46317,1731132929173, seqNum=5 because the exception is null or not the one we care about 2024-11-09T06:18:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T06:18:14,433 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:18:14,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/63aee012f706443cb4715580bc067af7 2024-11-09T06:18:14,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/b5593b66d528481a9e585c89e028f581 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5593b66d528481a9e585c89e028f581 2024-11-09T06:18:14,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5593b66d528481a9e585c89e028f581, entries=250, sequenceid=409, filesize=47.5 K 2024-11-09T06:18:14,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/79b8abdfd403498681fc61d87f4e2739 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/79b8abdfd403498681fc61d87f4e2739 2024-11-09T06:18:14,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/79b8abdfd403498681fc61d87f4e2739, entries=150, sequenceid=409, filesize=12.0 K 2024-11-09T06:18:14,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/63aee012f706443cb4715580bc067af7 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/63aee012f706443cb4715580bc067af7 2024-11-09T06:18:14,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/63aee012f706443cb4715580bc067af7, entries=150, sequenceid=409, filesize=12.0 K 2024-11-09T06:18:14,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 911ms, sequenceid=409, compaction requested=true 2024-11-09T06:18:14,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:14,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T06:18:14,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:14,637 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:18:14,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T06:18:14,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:14,637 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:18:14,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T06:18:14,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:18:14,638 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:18:14,638 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:18:14,638 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A is initiating minor compaction (all files) 2024-11-09T06:18:14,638 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B is initiating minor compaction (all files) 2024-11-09T06:18:14,638 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:14,638 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:14,638 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3246a628ce7242e5bbf3ab891a32fb66, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/ca382e07bf5f44bca6e7ba8ae4569a40, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/79b8abdfd403498681fc61d87f4e2739] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=36.9 K 2024-11-09T06:18:14,638 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6b21ddc5765e4396947e3464755a5773, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/580415f2cc6a4d6ab6a49d86c050b903, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5593b66d528481a9e585c89e028f581] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=109.4 K 2024-11-09T06:18:14,638 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:14,638 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. files: [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6b21ddc5765e4396947e3464755a5773, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/580415f2cc6a4d6ab6a49d86c050b903, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5593b66d528481a9e585c89e028f581] 2024-11-09T06:18:14,638 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3246a628ce7242e5bbf3ab891a32fb66, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731133091530 2024-11-09T06:18:14,639 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b21ddc5765e4396947e3464755a5773, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731133091530 2024-11-09T06:18:14,639 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting ca382e07bf5f44bca6e7ba8ae4569a40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731133091878 2024-11-09T06:18:14,639 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 580415f2cc6a4d6ab6a49d86c050b903, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731133091878 2024-11-09T06:18:14,639 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b8abdfd403498681fc61d87f4e2739, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731133092575 2024-11-09T06:18:14,639 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5593b66d528481a9e585c89e028f581, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731133092575 2024-11-09T06:18:14,643 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:14,644 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#B#compaction#562 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:14,644 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109d1ba1df729ca4469bfb08646dda8c78e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:14,644 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/23f35cfe80dc4139a424f8364ec6d8fc is 50, key is test_row_0/B:col10/1731133092576/Put/seqid=0 2024-11-09T06:18:14,652 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109d1ba1df729ca4469bfb08646dda8c78e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:14,652 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d1ba1df729ca4469bfb08646dda8c78e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:14,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742494_1670 (size=13255) 2024-11-09T06:18:14,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742495_1671 (size=4469) 2024-11-09T06:18:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46317 {}] regionserver.HRegion(8855): Flush requested on 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:14,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T06:18:14,884 INFO [Thread-2477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2477. 2024-11-09T06:18:14,884 DEBUG [Thread-2477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:18:14,884 INFO [Thread-2471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2471. 2024-11-09T06:18:14,884 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,884 DEBUG [Thread-2471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:18:14,884 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:14,884 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:14,884 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:14,884 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:14,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:14,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:14,885 INFO [Thread-2479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2479. 2024-11-09T06:18:14,885 DEBUG [Thread-2479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:18:14,885 DEBUG [Thread-2479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,885 DEBUG [Thread-2479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:14,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109eba570ad0f5c4dd59ea6f34b0f436e1d_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133093742/Put/seqid=0 2024-11-09T06:18:14,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742496_1672 (size=12454) 2024-11-09T06:18:14,894 INFO [Thread-2475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2475. 2024-11-09T06:18:14,894 DEBUG [Thread-2475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:18:14,894 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,894 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:14,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:15,064 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/23f35cfe80dc4139a424f8364ec6d8fc as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/23f35cfe80dc4139a424f8364ec6d8fc 2024-11-09T06:18:15,066 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#A#compaction#563 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:15,066 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/db0ba939e5614da580e1f00bd55b2954 is 175, key is test_row_0/A:col10/1731133092576/Put/seqid=0 2024-11-09T06:18:15,068 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into 23f35cfe80dc4139a424f8364ec6d8fc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:15,068 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:15,068 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B, priority=13, startTime=1731133094637; duration=0sec 2024-11-09T06:18:15,068 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T06:18:15,068 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:B 2024-11-09T06:18:15,068 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T06:18:15,069 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T06:18:15,069 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1541): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C is initiating minor compaction (all files) 2024-11-09T06:18:15,069 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C in TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:15,069 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b1c84dc61ac34413a3ed59fa31df5f5f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fd210f7915e84e369568450f5ca32f4e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/63aee012f706443cb4715580bc067af7] into tmpdir=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp, totalSize=36.9 K 2024-11-09T06:18:15,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742497_1673 (size=32209) 2024-11-09T06:18:15,069 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting b1c84dc61ac34413a3ed59fa31df5f5f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731133091530 2024-11-09T06:18:15,069 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting fd210f7915e84e369568450f5ca32f4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731133091878 2024-11-09T06:18:15,070 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] compactions.Compactor(225): Compacting 63aee012f706443cb4715580bc067af7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731133092575 2024-11-09T06:18:15,075 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aeacadfe7a3b9ebc8c8b0e6e39a10ca#C#compaction#565 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T06:18:15,075 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/b404b27dcd36466fb36c96f834ce917f is 50, key is test_row_0/C:col10/1731133092576/Put/seqid=0 2024-11-09T06:18:15,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742498_1674 (size=13255) 2024-11-09T06:18:15,294 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:15,297 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109eba570ad0f5c4dd59ea6f34b0f436e1d_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eba570ad0f5c4dd59ea6f34b0f436e1d_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:15,297 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/99d3fa500d5746ffaa05a605633220b4, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:15,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/99d3fa500d5746ffaa05a605633220b4 is 175, key is test_row_0/A:col10/1731133093742/Put/seqid=0 2024-11-09T06:18:15,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742499_1675 (size=31255) 2024-11-09T06:18:15,473 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/db0ba939e5614da580e1f00bd55b2954 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/db0ba939e5614da580e1f00bd55b2954 2024-11-09T06:18:15,476 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into db0ba939e5614da580e1f00bd55b2954(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:15,476 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:15,476 INFO [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A, priority=13, startTime=1731133094637; duration=0sec 2024-11-09T06:18:15,476 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:15,476 DEBUG [RS:0;4a84bcce7bad:46317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:A 2024-11-09T06:18:15,480 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/b404b27dcd36466fb36c96f834ce917f as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b404b27dcd36466fb36c96f834ce917f 2024-11-09T06:18:15,483 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C of 1aeacadfe7a3b9ebc8c8b0e6e39a10ca into b404b27dcd36466fb36c96f834ce917f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T06:18:15,483 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:15,483 INFO [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., storeName=1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C, priority=13, startTime=1731133094637; duration=0sec 2024-11-09T06:18:15,484 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T06:18:15,484 DEBUG [RS:0;4a84bcce7bad:46317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aeacadfe7a3b9ebc8c8b0e6e39a10ca:C 2024-11-09T06:18:15,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=425, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/99d3fa500d5746ffaa05a605633220b4 2024-11-09T06:18:15,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/67e2b037f24f414cbaae1b4b602b0c25 is 50, key is test_row_0/B:col10/1731133093742/Put/seqid=0 2024-11-09T06:18:15,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742500_1676 (size=12301) 2024-11-09T06:18:16,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/67e2b037f24f414cbaae1b4b602b0c25 2024-11-09T06:18:16,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/fb2c53ea45a843169635229ae6ba1c42 is 50, key is test_row_0/C:col10/1731133093742/Put/seqid=0 2024-11-09T06:18:16,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742501_1677 (size=12301) 2024-11-09T06:18:16,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/fb2c53ea45a843169635229ae6ba1c42 2024-11-09T06:18:16,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/99d3fa500d5746ffaa05a605633220b4 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/99d3fa500d5746ffaa05a605633220b4 2024-11-09T06:18:16,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/99d3fa500d5746ffaa05a605633220b4, entries=150, sequenceid=425, filesize=30.5 K 2024-11-09T06:18:16,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/67e2b037f24f414cbaae1b4b602b0c25 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/67e2b037f24f414cbaae1b4b602b0c25 2024-11-09T06:18:16,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/67e2b037f24f414cbaae1b4b602b0c25, entries=150, sequenceid=425, filesize=12.0 K 2024-11-09T06:18:16,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/fb2c53ea45a843169635229ae6ba1c42 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fb2c53ea45a843169635229ae6ba1c42 2024-11-09T06:18:16,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fb2c53ea45a843169635229ae6ba1c42, entries=150, sequenceid=425, filesize=12.0 K 2024-11-09T06:18:16,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=6.71 KB/6870 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 1651ms, sequenceid=425, compaction requested=false 2024-11-09T06:18:16,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: 2024-11-09T06:18:18,074 INFO [Thread-2473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2473. 2024-11-09T06:18:18,074 DEBUG [Thread-2473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T06:18:18,074 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:18,074 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:18,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:18,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T06:18:18,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-09T06:18:18,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-09T06:18:18,075 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-09T06:18:18,075 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-09T06:18:18,075 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-09T06:18:18,075 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T06:18:18,075 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2611 2024-11-09T06:18:18,075 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2709 2024-11-09T06:18:18,075 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2570 2024-11-09T06:18:18,075 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2614 2024-11-09T06:18:18,075 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2713 2024-11-09T06:18:18,075 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T06:18:18,075 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T06:18:18,075 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T06:18:18,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:18,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:18,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:18,076 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T06:18:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T06:18:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T06:18:18,078 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133098078"}]},"ts":"1731133098078"} 2024-11-09T06:18:18,080 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T06:18:18,080 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T06:18:18,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T06:18:18,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, UNASSIGN}] 2024-11-09T06:18:18,082 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, UNASSIGN 2024-11-09T06:18:18,082 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, regionState=CLOSING, regionLocation=4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:18,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, UNASSIGN because future has completed 2024-11-09T06:18:18,084 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T06:18:18,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173}] 2024-11-09T06:18:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T06:18:18,236 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:18,236 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T06:18:18,236 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, disabling compactions & flushes 2024-11-09T06:18:18,236 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:18,236 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:18,236 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. after waiting 0 ms 2024-11-09T06:18:18,236 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:18,236 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-09T06:18:18,236 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=A 2024-11-09T06:18:18,236 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:18,236 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=B 2024-11-09T06:18:18,236 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:18,236 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, store=C 2024-11-09T06:18:18,237 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T06:18:18,241 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cb43b2746401458883c38d6fb3b89dd0_1aeacadfe7a3b9ebc8c8b0e6e39a10ca is 50, key is test_row_0/A:col10/1731133094893/Put/seqid=0 2024-11-09T06:18:18,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742502_1678 (size=9914) 2024-11-09T06:18:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T06:18:18,644 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T06:18:18,647 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cb43b2746401458883c38d6fb3b89dd0_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cb43b2746401458883c38d6fb3b89dd0_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:18,647 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/060da74d462c4400a5eb8e0abb83e899, store: [table=TestAcidGuarantees family=A region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca] 2024-11-09T06:18:18,648 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/060da74d462c4400a5eb8e0abb83e899 is 175, key is test_row_0/A:col10/1731133094893/Put/seqid=0 2024-11-09T06:18:18,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742503_1679 (size=22561) 2024-11-09T06:18:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T06:18:19,051 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/060da74d462c4400a5eb8e0abb83e899 2024-11-09T06:18:19,056 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/cde16ec32a174e429e5c84bc87ef3110 is 50, key is test_row_0/B:col10/1731133094893/Put/seqid=0 2024-11-09T06:18:19,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742504_1680 (size=9857) 2024-11-09T06:18:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T06:18:19,459 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/cde16ec32a174e429e5c84bc87ef3110 2024-11-09T06:18:19,469 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/fbf8492c379d4a6da0850763c90c456e is 50, key is test_row_0/C:col10/1731133094893/Put/seqid=0 2024-11-09T06:18:19,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742505_1681 (size=9857) 2024-11-09T06:18:19,890 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/fbf8492c379d4a6da0850763c90c456e 2024-11-09T06:18:19,893 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/A/060da74d462c4400a5eb8e0abb83e899 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/060da74d462c4400a5eb8e0abb83e899 2024-11-09T06:18:19,895 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/060da74d462c4400a5eb8e0abb83e899, entries=100, sequenceid=433, filesize=22.0 K 2024-11-09T06:18:19,896 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/B/cde16ec32a174e429e5c84bc87ef3110 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/cde16ec32a174e429e5c84bc87ef3110 2024-11-09T06:18:19,898 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/cde16ec32a174e429e5c84bc87ef3110, entries=100, sequenceid=433, filesize=9.6 K 2024-11-09T06:18:19,898 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/.tmp/C/fbf8492c379d4a6da0850763c90c456e as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fbf8492c379d4a6da0850763c90c456e 2024-11-09T06:18:19,901 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fbf8492c379d4a6da0850763c90c456e, entries=100, sequenceid=433, filesize=9.6 K 2024-11-09T06:18:19,902 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 1666ms, sequenceid=433, compaction requested=true 2024-11-09T06:18:19,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/89b73bf2881d4eb4aeec9a548f9dd28e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8c7c35d7c421402cb4c6ee33b99c624f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/46e21a9f09ed4c6f9d3fa61e3b722cae, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5e715a336937476a9dac47598d8bad3a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/7413fa0178b84aa4a55563b7600639d5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/acc6f9c1983f47e5b4049aee42356f6c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4cdbb0bc1801477a8ed0b37e36b33fe3, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/9f0bc5a9027448c6ba6ad767b5ffc751, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/465dcbc377194856806773d5b1c4d295, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/afe90f4d727f423ba5ed45d384819de4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1a1c6e8fc01744f8824ed192c216f3c4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cd414e6c8bfa4e26bb72534334456e25, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/180a6d75d42546db92a00bff108a2af9, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/fe94e77786b442ca9cb353e3f9e70376, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b40d0ed444964f60a198429d69ba5f3e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/f69de498cdb64bee8986c9c8d8ddbac8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6f8e28b001924dbcaced9aef2a429198, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1cf6b74977a2443ebe8e920ac0714046, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5b8ff8bd65a04cc68b6023641bacefda, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cb286d38c7ad43aca8c7fa443dbf46e7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/09d20d5284004764b1d92935ded3b316, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/557c424be65346078b29da24d7dc85cc, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4c6b45da6341406481ebbbdc96ecfe27, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8d9b4a0876df4e70ba805c91df7d5445, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5f66a422c67400891fdcf8a83a8997f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6b21ddc5765e4396947e3464755a5773, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/580415f2cc6a4d6ab6a49d86c050b903, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5593b66d528481a9e585c89e028f581] to archive 2024-11-09T06:18:19,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:18:19,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/89b73bf2881d4eb4aeec9a548f9dd28e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/89b73bf2881d4eb4aeec9a548f9dd28e 2024-11-09T06:18:19,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8c7c35d7c421402cb4c6ee33b99c624f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8c7c35d7c421402cb4c6ee33b99c624f 2024-11-09T06:18:19,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/46e21a9f09ed4c6f9d3fa61e3b722cae to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/46e21a9f09ed4c6f9d3fa61e3b722cae 2024-11-09T06:18:19,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5e715a336937476a9dac47598d8bad3a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5e715a336937476a9dac47598d8bad3a 2024-11-09T06:18:19,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/7413fa0178b84aa4a55563b7600639d5 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/7413fa0178b84aa4a55563b7600639d5 2024-11-09T06:18:19,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/acc6f9c1983f47e5b4049aee42356f6c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/acc6f9c1983f47e5b4049aee42356f6c 2024-11-09T06:18:19,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4cdbb0bc1801477a8ed0b37e36b33fe3 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4cdbb0bc1801477a8ed0b37e36b33fe3 2024-11-09T06:18:19,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/9f0bc5a9027448c6ba6ad767b5ffc751 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/9f0bc5a9027448c6ba6ad767b5ffc751 2024-11-09T06:18:19,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/465dcbc377194856806773d5b1c4d295 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/465dcbc377194856806773d5b1c4d295 2024-11-09T06:18:19,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/afe90f4d727f423ba5ed45d384819de4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/afe90f4d727f423ba5ed45d384819de4 2024-11-09T06:18:19,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1a1c6e8fc01744f8824ed192c216f3c4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1a1c6e8fc01744f8824ed192c216f3c4 2024-11-09T06:18:19,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cd414e6c8bfa4e26bb72534334456e25 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cd414e6c8bfa4e26bb72534334456e25 2024-11-09T06:18:19,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/180a6d75d42546db92a00bff108a2af9 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/180a6d75d42546db92a00bff108a2af9 2024-11-09T06:18:19,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/fe94e77786b442ca9cb353e3f9e70376 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/fe94e77786b442ca9cb353e3f9e70376 2024-11-09T06:18:19,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b40d0ed444964f60a198429d69ba5f3e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b40d0ed444964f60a198429d69ba5f3e 2024-11-09T06:18:19,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/f69de498cdb64bee8986c9c8d8ddbac8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/f69de498cdb64bee8986c9c8d8ddbac8 2024-11-09T06:18:19,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6f8e28b001924dbcaced9aef2a429198 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6f8e28b001924dbcaced9aef2a429198 2024-11-09T06:18:19,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1cf6b74977a2443ebe8e920ac0714046 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/1cf6b74977a2443ebe8e920ac0714046 2024-11-09T06:18:19,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5b8ff8bd65a04cc68b6023641bacefda to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/5b8ff8bd65a04cc68b6023641bacefda 2024-11-09T06:18:19,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cb286d38c7ad43aca8c7fa443dbf46e7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/cb286d38c7ad43aca8c7fa443dbf46e7 2024-11-09T06:18:19,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/09d20d5284004764b1d92935ded3b316 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/09d20d5284004764b1d92935ded3b316 2024-11-09T06:18:19,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/557c424be65346078b29da24d7dc85cc to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/557c424be65346078b29da24d7dc85cc 2024-11-09T06:18:19,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4c6b45da6341406481ebbbdc96ecfe27 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/4c6b45da6341406481ebbbdc96ecfe27 2024-11-09T06:18:19,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8d9b4a0876df4e70ba805c91df7d5445 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/8d9b4a0876df4e70ba805c91df7d5445 2024-11-09T06:18:19,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5f66a422c67400891fdcf8a83a8997f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5f66a422c67400891fdcf8a83a8997f 2024-11-09T06:18:19,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6b21ddc5765e4396947e3464755a5773 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/6b21ddc5765e4396947e3464755a5773 2024-11-09T06:18:19,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/580415f2cc6a4d6ab6a49d86c050b903 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/580415f2cc6a4d6ab6a49d86c050b903 2024-11-09T06:18:19,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5593b66d528481a9e585c89e028f581 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/b5593b66d528481a9e585c89e028f581 2024-11-09T06:18:19,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3f6ab4c778b945ba8aed0919558363c7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/673a99b250b5427bb5169ae286aa85cf, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/165d4bd0a1de4e7098c0fb78feaa8f5e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5231459abda84366888a230e010c1fc7, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/36dfd8556eed42dd979c730f91df57d6, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/acd4057451964ec783644c39368b17a8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a1894d2139c64161aa20bcb7689c910f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/52f78ad049c440a8b37feba117a18e34, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/2938f6d80c2d40f7ade1a3566223f167, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/32cadad2c1b244bc91bde122d6ad805c, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/928e879bdca5434b8514d05c34fbea77, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/bde4934ea4c6425ea3b56109acd8c5ea, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/aee15b5ec6d54450891ba775c095ec1b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/1deb2698a61e487386ff1342d187d506, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c8d88991b4d14df5bdf9bed936d5438a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3891b077e74840f3ac4d151bca820ed0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/0094cb5f9d484f66b2cadcf6d3e36d44, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a4dad32abd884377bde9ef018f347785, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/506d5fda1471434d9b445a1ded801177, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/b46ed9aa57f348089ecc4a92535a7421, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a18733bc4c9b424da6b9f2ad91fd0beb, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5dd6405de56d40c9937a4509ef4599c8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c33b8b4f5c054cf59911e662d026293a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/2294adb4056f4d019a56a38b6fb08b4e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3246a628ce7242e5bbf3ab891a32fb66, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/9963025bde7547788a586d49cddda777, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/ca382e07bf5f44bca6e7ba8ae4569a40, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/79b8abdfd403498681fc61d87f4e2739] to archive 2024-11-09T06:18:19,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:18:19,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3f6ab4c778b945ba8aed0919558363c7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3f6ab4c778b945ba8aed0919558363c7 2024-11-09T06:18:19,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/673a99b250b5427bb5169ae286aa85cf to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/673a99b250b5427bb5169ae286aa85cf 2024-11-09T06:18:19,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/165d4bd0a1de4e7098c0fb78feaa8f5e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/165d4bd0a1de4e7098c0fb78feaa8f5e 2024-11-09T06:18:19,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5231459abda84366888a230e010c1fc7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5231459abda84366888a230e010c1fc7 2024-11-09T06:18:19,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/36dfd8556eed42dd979c730f91df57d6 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/36dfd8556eed42dd979c730f91df57d6 2024-11-09T06:18:19,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/acd4057451964ec783644c39368b17a8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/acd4057451964ec783644c39368b17a8 2024-11-09T06:18:19,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a1894d2139c64161aa20bcb7689c910f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a1894d2139c64161aa20bcb7689c910f 2024-11-09T06:18:19,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/52f78ad049c440a8b37feba117a18e34 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/52f78ad049c440a8b37feba117a18e34 2024-11-09T06:18:19,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/2938f6d80c2d40f7ade1a3566223f167 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/2938f6d80c2d40f7ade1a3566223f167 2024-11-09T06:18:19,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/32cadad2c1b244bc91bde122d6ad805c to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/32cadad2c1b244bc91bde122d6ad805c 2024-11-09T06:18:19,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/928e879bdca5434b8514d05c34fbea77 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/928e879bdca5434b8514d05c34fbea77 2024-11-09T06:18:19,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/bde4934ea4c6425ea3b56109acd8c5ea to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/bde4934ea4c6425ea3b56109acd8c5ea 2024-11-09T06:18:19,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/aee15b5ec6d54450891ba775c095ec1b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/aee15b5ec6d54450891ba775c095ec1b 2024-11-09T06:18:19,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/1deb2698a61e487386ff1342d187d506 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/1deb2698a61e487386ff1342d187d506 2024-11-09T06:18:19,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c8d88991b4d14df5bdf9bed936d5438a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c8d88991b4d14df5bdf9bed936d5438a 2024-11-09T06:18:19,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3891b077e74840f3ac4d151bca820ed0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3891b077e74840f3ac4d151bca820ed0 2024-11-09T06:18:19,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/0094cb5f9d484f66b2cadcf6d3e36d44 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/0094cb5f9d484f66b2cadcf6d3e36d44 2024-11-09T06:18:19,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a4dad32abd884377bde9ef018f347785 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a4dad32abd884377bde9ef018f347785 2024-11-09T06:18:19,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/506d5fda1471434d9b445a1ded801177 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/506d5fda1471434d9b445a1ded801177 2024-11-09T06:18:19,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/b46ed9aa57f348089ecc4a92535a7421 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/b46ed9aa57f348089ecc4a92535a7421 2024-11-09T06:18:19,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a18733bc4c9b424da6b9f2ad91fd0beb to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/a18733bc4c9b424da6b9f2ad91fd0beb 2024-11-09T06:18:19,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5dd6405de56d40c9937a4509ef4599c8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/5dd6405de56d40c9937a4509ef4599c8 2024-11-09T06:18:19,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c33b8b4f5c054cf59911e662d026293a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/c33b8b4f5c054cf59911e662d026293a 2024-11-09T06:18:19,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/2294adb4056f4d019a56a38b6fb08b4e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/2294adb4056f4d019a56a38b6fb08b4e 2024-11-09T06:18:19,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3246a628ce7242e5bbf3ab891a32fb66 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/3246a628ce7242e5bbf3ab891a32fb66 2024-11-09T06:18:19,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/9963025bde7547788a586d49cddda777 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/9963025bde7547788a586d49cddda777 2024-11-09T06:18:19,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/ca382e07bf5f44bca6e7ba8ae4569a40 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/ca382e07bf5f44bca6e7ba8ae4569a40 2024-11-09T06:18:19,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/79b8abdfd403498681fc61d87f4e2739 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/79b8abdfd403498681fc61d87f4e2739 2024-11-09T06:18:19,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/d2b6fd5f015943d2ba594900d2f46b04, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5aaa61d3d0fa493a9400bd32c0874e9b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/1c54e2745eaf4178a680fd4aa5d3eced, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/1bfbd64c79d748a5bc774ef96c6a4182, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c22f4734187c48d3a17bf008290022a0, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/cfc4f5e162774545aef95f54725163e5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/0fb9e7d6d6a241738f13bbef99090d2f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/89298c053b724edc98f5d336b597d053, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7b90fe6d725b4cf5aaad058f6f773048, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7b52d1af95344543bda8122cc1ca41af, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7cc163c917c94f87897dab982da693ba, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a3ad618400e74e318ff5016c903311a8, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/106d5c376d4349249d011c348540b749, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a0533f4dac9340348011268168e04f16, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a434d2b8d8e14609ba9e6db074123f2f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/8829207263bd4411ba4faf158259740b, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/95e408dc5bc343f7ba6b126460d3b638, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b0a6eb0d0e2c4413b19ce675922bbb3a, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/9f39759b21c446f99b0399928c3049a4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/cc19a363f12e4bd59a362abb8b1e12e4, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c4bea2b2583745dcb4d55c04cbed95ce, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5378cc909b0c4bcd8507af73f6edb010, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/77bee433a9b2461cb9d5046c32aeacec, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/612a35f6c7b74e21837cc675371788b5, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b1c84dc61ac34413a3ed59fa31df5f5f, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/e50811313c6b499d959baef9602d6f96, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fd210f7915e84e369568450f5ca32f4e, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/63aee012f706443cb4715580bc067af7] to archive 2024-11-09T06:18:19,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T06:18:19,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/d2b6fd5f015943d2ba594900d2f46b04 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/d2b6fd5f015943d2ba594900d2f46b04 2024-11-09T06:18:19,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5aaa61d3d0fa493a9400bd32c0874e9b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5aaa61d3d0fa493a9400bd32c0874e9b 2024-11-09T06:18:19,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/1c54e2745eaf4178a680fd4aa5d3eced to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/1c54e2745eaf4178a680fd4aa5d3eced 2024-11-09T06:18:19,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/1bfbd64c79d748a5bc774ef96c6a4182 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/1bfbd64c79d748a5bc774ef96c6a4182 2024-11-09T06:18:19,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c22f4734187c48d3a17bf008290022a0 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c22f4734187c48d3a17bf008290022a0 2024-11-09T06:18:19,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/cfc4f5e162774545aef95f54725163e5 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/cfc4f5e162774545aef95f54725163e5 2024-11-09T06:18:19,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/0fb9e7d6d6a241738f13bbef99090d2f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/0fb9e7d6d6a241738f13bbef99090d2f 2024-11-09T06:18:19,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/89298c053b724edc98f5d336b597d053 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/89298c053b724edc98f5d336b597d053 2024-11-09T06:18:19,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7b90fe6d725b4cf5aaad058f6f773048 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7b90fe6d725b4cf5aaad058f6f773048 2024-11-09T06:18:19,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7b52d1af95344543bda8122cc1ca41af to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7b52d1af95344543bda8122cc1ca41af 2024-11-09T06:18:19,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7cc163c917c94f87897dab982da693ba to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/7cc163c917c94f87897dab982da693ba 2024-11-09T06:18:19,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a3ad618400e74e318ff5016c903311a8 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a3ad618400e74e318ff5016c903311a8 2024-11-09T06:18:19,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/106d5c376d4349249d011c348540b749 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/106d5c376d4349249d011c348540b749 2024-11-09T06:18:19,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a0533f4dac9340348011268168e04f16 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a0533f4dac9340348011268168e04f16 2024-11-09T06:18:19,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a434d2b8d8e14609ba9e6db074123f2f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/a434d2b8d8e14609ba9e6db074123f2f 2024-11-09T06:18:19,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/8829207263bd4411ba4faf158259740b to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/8829207263bd4411ba4faf158259740b 2024-11-09T06:18:19,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/95e408dc5bc343f7ba6b126460d3b638 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/95e408dc5bc343f7ba6b126460d3b638 2024-11-09T06:18:19,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b0a6eb0d0e2c4413b19ce675922bbb3a to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b0a6eb0d0e2c4413b19ce675922bbb3a 2024-11-09T06:18:19,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/9f39759b21c446f99b0399928c3049a4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/9f39759b21c446f99b0399928c3049a4 2024-11-09T06:18:19,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/cc19a363f12e4bd59a362abb8b1e12e4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/cc19a363f12e4bd59a362abb8b1e12e4 2024-11-09T06:18:19,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c4bea2b2583745dcb4d55c04cbed95ce to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/c4bea2b2583745dcb4d55c04cbed95ce 2024-11-09T06:18:19,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5378cc909b0c4bcd8507af73f6edb010 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/5378cc909b0c4bcd8507af73f6edb010 2024-11-09T06:18:19,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/77bee433a9b2461cb9d5046c32aeacec to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/77bee433a9b2461cb9d5046c32aeacec 2024-11-09T06:18:19,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/612a35f6c7b74e21837cc675371788b5 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/612a35f6c7b74e21837cc675371788b5 2024-11-09T06:18:19,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b1c84dc61ac34413a3ed59fa31df5f5f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b1c84dc61ac34413a3ed59fa31df5f5f 2024-11-09T06:18:19,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/e50811313c6b499d959baef9602d6f96 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/e50811313c6b499d959baef9602d6f96 2024-11-09T06:18:19,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fd210f7915e84e369568450f5ca32f4e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fd210f7915e84e369568450f5ca32f4e 2024-11-09T06:18:19,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/63aee012f706443cb4715580bc067af7 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/63aee012f706443cb4715580bc067af7 2024-11-09T06:18:19,981 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=4 2024-11-09T06:18:19,981 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. 2024-11-09T06:18:19,981 DEBUG [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca: Waiting for close lock at 1731133098236Running coprocessor pre-close hooks at 1731133098236Disabling compacts and flushes for region at 1731133098236Disabling writes for close at 1731133098236Obtaining lock to block concurrent updates at 1731133098236Preparing flush snapshotting stores in 1aeacadfe7a3b9ebc8c8b0e6e39a10ca at 1731133098236Finished memstore snapshotting TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731133098237 (+1 ms)Flushing stores of TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca. at 1731133098237Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A: creating writer at 1731133098237Flushing mob file 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A: appending metadata at 1731133098241 (+4 ms)Flushing mob file 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A: closing flushed file at 1731133098241Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A: appending metadata at 1731133098647 (+406 ms)Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A: closing flushed file at 1731133098648 (+1 ms)Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B: creating writer at 1731133099054 (+406 ms)Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B: appending metadata at 1731133099056 (+2 ms)Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B: closing flushed file at 1731133099056Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C: creating writer at 1731133099462 (+406 ms)Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C: appending metadata at 1731133099469 (+7 ms)Flushing 1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C: closing flushed file at 1731133099469Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@487992c0: reopening flushed file at 1731133099892 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d2686b6: reopening flushed file at 1731133099895 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@385031d2: reopening flushed file at 1731133099898 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 1aeacadfe7a3b9ebc8c8b0e6e39a10ca in 1666ms, sequenceid=433, compaction requested=true at 1731133099902 (+4 ms)Writing region close event to WAL at 1731133099978 (+76 ms)Running coprocessor post-close hooks at 1731133099981 (+3 ms)Closed at 1731133099981 2024-11-09T06:18:19,983 INFO [RS_CLOSE_REGION-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed 1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:19,983 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, regionState=CLOSED 2024-11-09T06:18:19,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 because future has completed 2024-11-09T06:18:19,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-09T06:18:19,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, server=4a84bcce7bad,46317,1731132929173 in 1.9010 sec 2024-11-09T06:18:19,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-09T06:18:19,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aeacadfe7a3b9ebc8c8b0e6e39a10ca, UNASSIGN in 1.9060 sec 2024-11-09T06:18:19,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-09T06:18:19,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9080 sec 2024-11-09T06:18:19,991 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731133099991"}]},"ts":"1731133099991"} 2024-11-09T06:18:19,992 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T06:18:19,992 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T06:18:19,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9170 sec 2024-11-09T06:18:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T06:18:20,223 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:18:20,224 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T06:18:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:18:20,225 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:18:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T06:18:20,226 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:18:20,228 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,229 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C, FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/recovered.edits] 2024-11-09T06:18:20,231 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/060da74d462c4400a5eb8e0abb83e899 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/060da74d462c4400a5eb8e0abb83e899 2024-11-09T06:18:20,232 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/99d3fa500d5746ffaa05a605633220b4 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/99d3fa500d5746ffaa05a605633220b4 2024-11-09T06:18:20,233 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/db0ba939e5614da580e1f00bd55b2954 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/A/db0ba939e5614da580e1f00bd55b2954 2024-11-09T06:18:20,234 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/23f35cfe80dc4139a424f8364ec6d8fc to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/23f35cfe80dc4139a424f8364ec6d8fc 2024-11-09T06:18:20,235 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/67e2b037f24f414cbaae1b4b602b0c25 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/67e2b037f24f414cbaae1b4b602b0c25 2024-11-09T06:18:20,236 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/cde16ec32a174e429e5c84bc87ef3110 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/B/cde16ec32a174e429e5c84bc87ef3110 2024-11-09T06:18:20,238 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b404b27dcd36466fb36c96f834ce917f to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/b404b27dcd36466fb36c96f834ce917f 2024-11-09T06:18:20,238 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fb2c53ea45a843169635229ae6ba1c42 to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fb2c53ea45a843169635229ae6ba1c42 2024-11-09T06:18:20,239 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fbf8492c379d4a6da0850763c90c456e to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/C/fbf8492c379d4a6da0850763c90c456e 2024-11-09T06:18:20,241 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/recovered.edits/436.seqid to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca/recovered.edits/436.seqid 2024-11-09T06:18:20,241 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/default/TestAcidGuarantees/1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,241 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T06:18:20,242 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T06:18:20,242 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T06:18:20,245 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090acdf77598454e5ab3729c7d08e6ad39_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090acdf77598454e5ab3729c7d08e6ad39_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,246 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091c4611e68967405c8bceb5dd2650f500_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091c4611e68967405c8bceb5dd2650f500_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,246 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091d4ad233dfd849cbae34eb52f2cee875_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091d4ad233dfd849cbae34eb52f2cee875_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,247 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109204a8efe82cf40ea80d82933b41e2eef_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109204a8efe82cf40ea80d82933b41e2eef_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,248 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092728344da9b8474693258d272d03c27d_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092728344da9b8474693258d272d03c27d_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,249 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109297dacd9cfdf446997fc62f7b1f235d7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109297dacd9cfdf446997fc62f7b1f235d7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,250 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110932b0e752149b4455ad6bcf1efcec64a8_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110932b0e752149b4455ad6bcf1efcec64a8_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,251 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093e84c74da477443b91001c58809124b2_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093e84c74da477443b91001c58809124b2_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,252 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110968dc907c74e6407ea6e65f9f08c7086a_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110968dc907c74e6407ea6e65f9f08c7086a_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,252 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096b504310eb5947bd95ec330a5dcecdca_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096b504310eb5947bd95ec330a5dcecdca_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,253 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110976d3c6b18081445ca2cc19a5c5a0e3f2_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110976d3c6b18081445ca2cc19a5c5a0e3f2_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,254 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110987d56f195f7044a98e387a2e2ae8b370_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110987d56f195f7044a98e387a2e2ae8b370_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,255 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099732a93a6427474baba7f92023cd730b_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099732a93a6427474baba7f92023cd730b_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,255 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099fff183fbc744c07ab09d933f2a4782e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099fff183fbc744c07ab09d933f2a4782e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,256 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b6a910450acb4084a03da664ff0a28a7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b6a910450acb4084a03da664ff0a28a7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,257 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ba53a310437947aabe9c42acdfef4507_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ba53a310437947aabe9c42acdfef4507_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,258 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cb43b2746401458883c38d6fb3b89dd0_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cb43b2746401458883c38d6fb3b89dd0_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,258 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d555c627e0ec4acea9be82e87fdd6c53_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d555c627e0ec4acea9be82e87fdd6c53_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,259 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dbb6eef2486840d99a25dd40394eaae7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dbb6eef2486840d99a25dd40394eaae7_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,260 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ded96b8194514d3dae0add5703f6d98e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ded96b8194514d3dae0add5703f6d98e_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,261 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e8edf643fd7548cfa178e08f7465b76c_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e8edf643fd7548cfa178e08f7465b76c_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,261 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eb634e11fe424da18403b26933ff96ac_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eb634e11fe424da18403b26933ff96ac_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,262 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eba570ad0f5c4dd59ea6f34b0f436e1d_1aeacadfe7a3b9ebc8c8b0e6e39a10ca to hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109eba570ad0f5c4dd59ea6f34b0f436e1d_1aeacadfe7a3b9ebc8c8b0e6e39a10ca 2024-11-09T06:18:20,263 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T06:18:20,264 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:18:20,266 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T06:18:20,268 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T06:18:20,269 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:18:20,269 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T06:18:20,269 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731133100269"}]},"ts":"9223372036854775807"} 2024-11-09T06:18:20,270 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T06:18:20,270 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1aeacadfe7a3b9ebc8c8b0e6e39a10ca, NAME => 'TestAcidGuarantees,,1731133071073.1aeacadfe7a3b9ebc8c8b0e6e39a10ca.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T06:18:20,270 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T06:18:20,271 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731133100270"}]},"ts":"9223372036854775807"} 2024-11-09T06:18:20,272 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T06:18:20,272 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T06:18:20,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-09T06:18:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37069 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T06:18:20,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T06:18:20,335 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T06:18:20,342 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 252), OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=655 (was 751), ProcessCount=11 (was 11), AvailableMemoryMB=454 (was 632) 2024-11-09T06:18:20,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T06:18:20,342 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T06:18:20,342 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T06:18:20,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:20,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:20,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T06:18:20,343 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T06:18:20,343 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=855483586, stopped=false 2024-11-09T06:18:20,343 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4a84bcce7bad,37069,1731132928314 2024-11-09T06:18:20,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T06:18:20,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T06:18:20,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:18:20,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:18:20,444 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T06:18:20,445 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T06:18:20,445 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T06:18:20,446 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T06:18:20,446 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T06:18:20,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:20,446 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4a84bcce7bad,46317,1731132929173' ***** 2024-11-09T06:18:20,446 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T06:18:20,446 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T06:18:20,447 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T06:18:20,447 INFO [RS:0;4a84bcce7bad:46317 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T06:18:20,447 INFO [regionserver/4a84bcce7bad:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T06:18:20,447 INFO [regionserver/4a84bcce7bad:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T06:18:20,447 INFO [RS:0;4a84bcce7bad:46317 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T06:18:20,447 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(959): stopping server 4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:20,447 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T06:18:20,447 INFO [RS:0;4a84bcce7bad:46317 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4a84bcce7bad:46317. 2024-11-09T06:18:20,447 DEBUG [RS:0;4a84bcce7bad:46317 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T06:18:20,447 DEBUG [RS:0;4a84bcce7bad:46317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:20,447 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T06:18:20,447 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T06:18:20,447 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T06:18:20,447 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T06:18:20,448 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-09T06:18:20,448 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-09T06:18:20,448 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T06:18:20,448 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T06:18:20,448 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T06:18:20,448 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T06:18:20,448 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T06:18:20,448 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T06:18:20,449 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-09T06:18:20,473 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/info/074ce9343e2a4c47b3c62feed59aaec8 is 95, key is TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f./info:/1731132961105/DeleteFamily/seqid=0 2024-11-09T06:18:20,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742506_1682 (size=6514) 2024-11-09T06:18:20,522 INFO [regionserver/4a84bcce7bad:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T06:18:20,648 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T06:18:20,849 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T06:18:20,887 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/info/074ce9343e2a4c47b3c62feed59aaec8 2024-11-09T06:18:20,905 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/ns/6f670176ed684ceb8a55800eea918f9b is 93, key is TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f./ns:/1731132961056/DeleteFamily/seqid=0 2024-11-09T06:18:20,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742507_1683 (size=5906) 2024-11-09T06:18:21,049 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T06:18:21,249 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T06:18:21,308 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/ns/6f670176ed684ceb8a55800eea918f9b 2024-11-09T06:18:21,328 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/rep_barrier/cbfcdf27017941798dc3b61f2cde5dc9 is 102, key is TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f./rep_barrier:/1731132961056/DeleteFamily/seqid=0 2024-11-09T06:18:21,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742508_1684 (size=6025) 2024-11-09T06:18:21,449 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T06:18:21,649 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T06:18:21,731 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/rep_barrier/cbfcdf27017941798dc3b61f2cde5dc9 2024-11-09T06:18:21,748 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/table/5f975325079243a5a5e051e1488d2049 is 96, key is TestAcidGuarantees,,1731132934245.4cec2ccd1a271d540c1f369d10f41e7f./table:/1731132961056/DeleteFamily/seqid=0 2024-11-09T06:18:21,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742509_1685 (size=5936) 2024-11-09T06:18:21,850 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T06:18:22,050 DEBUG [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T06:18:22,151 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/table/5f975325079243a5a5e051e1488d2049 2024-11-09T06:18:22,154 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/info/074ce9343e2a4c47b3c62feed59aaec8 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/info/074ce9343e2a4c47b3c62feed59aaec8 2024-11-09T06:18:22,157 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/info/074ce9343e2a4c47b3c62feed59aaec8, entries=12, sequenceid=90, filesize=6.4 K 2024-11-09T06:18:22,158 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/ns/6f670176ed684ceb8a55800eea918f9b as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/ns/6f670176ed684ceb8a55800eea918f9b 2024-11-09T06:18:22,161 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/ns/6f670176ed684ceb8a55800eea918f9b, entries=8, sequenceid=90, filesize=5.8 K 2024-11-09T06:18:22,162 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/rep_barrier/cbfcdf27017941798dc3b61f2cde5dc9 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/rep_barrier/cbfcdf27017941798dc3b61f2cde5dc9 2024-11-09T06:18:22,165 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/rep_barrier/cbfcdf27017941798dc3b61f2cde5dc9, entries=6, sequenceid=90, filesize=5.9 K 2024-11-09T06:18:22,165 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/.tmp/table/5f975325079243a5a5e051e1488d2049 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/table/5f975325079243a5a5e051e1488d2049 2024-11-09T06:18:22,168 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/table/5f975325079243a5a5e051e1488d2049, entries=7, sequenceid=90, filesize=5.8 K 2024-11-09T06:18:22,169 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1721ms, sequenceid=90, compaction requested=false 2024-11-09T06:18:22,188 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-09T06:18:22,189 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T06:18:22,189 INFO [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T06:18:22,189 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731133100448Running coprocessor pre-close hooks at 1731133100448Disabling compacts and flushes for region at 1731133100448Disabling writes for close at 1731133100448Obtaining lock to block concurrent updates at 1731133100449 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731133100449Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731133100449Flushing stores of hbase:meta,,1.1588230740 at 1731133100450 (+1 ms)Flushing 1588230740/info: creating writer at 1731133100450Flushing 1588230740/info: appending metadata at 1731133100473 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731133100473Flushing 1588230740/ns: creating writer at 1731133100891 (+418 ms)Flushing 1588230740/ns: appending metadata at 1731133100905 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731133100905Flushing 1588230740/rep_barrier: creating writer at 1731133101313 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731133101327 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731133101328 (+1 ms)Flushing 1588230740/table: creating writer at 1731133101734 (+406 ms)Flushing 1588230740/table: appending metadata at 1731133101748 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731133101748Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d824e31: reopening flushed file at 1731133102154 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21b626a4: reopening flushed file at 1731133102158 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c7744a8: reopening flushed file at 1731133102161 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3375b40: reopening flushed file at 1731133102165 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1721ms, sequenceid=90, compaction requested=false at 1731133102169 (+4 ms)Writing region close event to WAL at 1731133102185 (+16 ms)Running coprocessor post-close hooks at 1731133102188 (+3 ms)Closed at 1731133102189 (+1 ms) 2024-11-09T06:18:22,189 DEBUG [RS_CLOSE_META-regionserver/4a84bcce7bad:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T06:18:22,250 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(976): stopping server 4a84bcce7bad,46317,1731132929173; all regions closed. 2024-11-09T06:18:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741834_1010 (size=25344) 2024-11-09T06:18:22,256 DEBUG [RS:0;4a84bcce7bad:46317 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/oldWALs 2024-11-09T06:18:22,256 INFO [RS:0;4a84bcce7bad:46317 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4a84bcce7bad%2C46317%2C1731132929173.meta:.meta(num 1731132933650) 2024-11-09T06:18:22,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741832_1008 (size=17582145) 2024-11-09T06:18:22,260 DEBUG [RS:0;4a84bcce7bad:46317 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/oldWALs 2024-11-09T06:18:22,260 INFO [RS:0;4a84bcce7bad:46317 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4a84bcce7bad%2C46317%2C1731132929173:(num 1731132932641) 2024-11-09T06:18:22,260 DEBUG [RS:0;4a84bcce7bad:46317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T06:18:22,260 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T06:18:22,260 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T06:18:22,260 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.ChoreService(370): Chore service for: regionserver/4a84bcce7bad:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T06:18:22,260 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T06:18:22,260 INFO [regionserver/4a84bcce7bad:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T06:18:22,261 INFO [RS:0;4a84bcce7bad:46317 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46317 2024-11-09T06:18:22,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4a84bcce7bad,46317,1731132929173 2024-11-09T06:18:22,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T06:18:22,286 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T06:18:22,297 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4a84bcce7bad,46317,1731132929173] 2024-11-09T06:18:22,307 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4a84bcce7bad,46317,1731132929173 already deleted, retry=false 2024-11-09T06:18:22,307 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4a84bcce7bad,46317,1731132929173 expired; onlineServers=0 2024-11-09T06:18:22,307 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4a84bcce7bad,37069,1731132928314' ***** 2024-11-09T06:18:22,307 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T06:18:22,307 INFO [M:0;4a84bcce7bad:37069 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T06:18:22,308 INFO [M:0;4a84bcce7bad:37069 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T06:18:22,308 DEBUG [M:0;4a84bcce7bad:37069 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T06:18:22,308 DEBUG [M:0;4a84bcce7bad:37069 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T06:18:22,308 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T06:18:22,308 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster-HFileCleaner.large.0-1731132932298 {}] cleaner.HFileCleaner(306): Exit Thread[master/4a84bcce7bad:0:becomeActiveMaster-HFileCleaner.large.0-1731132932298,5,FailOnTimeoutGroup] 2024-11-09T06:18:22,308 DEBUG [master/4a84bcce7bad:0:becomeActiveMaster-HFileCleaner.small.0-1731132932301 {}] cleaner.HFileCleaner(306): Exit Thread[master/4a84bcce7bad:0:becomeActiveMaster-HFileCleaner.small.0-1731132932301,5,FailOnTimeoutGroup] 2024-11-09T06:18:22,308 INFO [M:0;4a84bcce7bad:37069 {}] hbase.ChoreService(370): Chore service for: master/4a84bcce7bad:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T06:18:22,308 INFO [M:0;4a84bcce7bad:37069 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T06:18:22,308 DEBUG [M:0;4a84bcce7bad:37069 {}] master.HMaster(1795): Stopping service threads 2024-11-09T06:18:22,308 INFO [M:0;4a84bcce7bad:37069 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T06:18:22,309 INFO [M:0;4a84bcce7bad:37069 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T06:18:22,309 ERROR [M:0;4a84bcce7bad:37069 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:43365 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:43365,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-09T06:18:22,309 INFO [M:0;4a84bcce7bad:37069 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T06:18:22,310 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T06:18:22,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T06:18:22,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T06:18:22,318 DEBUG [M:0;4a84bcce7bad:37069 {}] zookeeper.ZKUtil(347): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T06:18:22,318 WARN [M:0;4a84bcce7bad:37069 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T06:18:22,319 INFO [M:0;4a84bcce7bad:37069 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/.lastflushedseqids 2024-11-09T06:18:22,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742510_1686 (size=72) 2024-11-09T06:18:22,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T06:18:22,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46317-0x1011e3c29400001, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T06:18:22,398 INFO [RS:0;4a84bcce7bad:46317 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T06:18:22,398 INFO [RS:0;4a84bcce7bad:46317 {}] regionserver.HRegionServer(1031): Exiting; stopping=4a84bcce7bad,46317,1731132929173; zookeeper connection closed. 2024-11-09T06:18:22,399 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@48101e2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@48101e2 2024-11-09T06:18:22,399 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T06:18:22,730 INFO [M:0;4a84bcce7bad:37069 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T06:18:22,730 INFO [M:0;4a84bcce7bad:37069 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T06:18:22,730 DEBUG [M:0;4a84bcce7bad:37069 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T06:18:22,730 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T06:18:22,730 DEBUG [M:0;4a84bcce7bad:37069 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T06:18:22,730 DEBUG [M:0;4a84bcce7bad:37069 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T06:18:22,730 DEBUG [M:0;4a84bcce7bad:37069 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T06:18:22,730 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=774.35 KB heapSize=948.59 KB 2024-11-09T06:18:22,749 DEBUG [M:0;4a84bcce7bad:37069 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09667b6b40cb47b6a1f5876106eaa8d2 is 82, key is hbase:meta,,1/info:regioninfo/1731132933780/Put/seqid=0 2024-11-09T06:18:22,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742511_1687 (size=5672) 2024-11-09T06:18:23,154 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09667b6b40cb47b6a1f5876106eaa8d2 2024-11-09T06:18:23,174 DEBUG [M:0;4a84bcce7bad:37069 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4331b7c4bb0e42aa990b2485b82c8080 is 2480, key is \x00\x00\x00\x00\x00\x00\x00T/proc:d/1731133022263/Put/seqid=0 2024-11-09T06:18:23,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742512_1688 (size=44379) 2024-11-09T06:18:23,590 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=773.80 KB at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4331b7c4bb0e42aa990b2485b82c8080 2024-11-09T06:18:23,593 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4331b7c4bb0e42aa990b2485b82c8080 2024-11-09T06:18:23,611 DEBUG [M:0;4a84bcce7bad:37069 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d6053347b6ea42779199f5dbfcf176bb is 69, key is 4a84bcce7bad,46317,1731132929173/rs:state/1731132932331/Put/seqid=0 2024-11-09T06:18:23,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073742513_1689 (size=5156) 2024-11-09T06:18:23,614 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d6053347b6ea42779199f5dbfcf176bb 2024-11-09T06:18:23,618 DEBUG [M:0;4a84bcce7bad:37069 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09667b6b40cb47b6a1f5876106eaa8d2 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09667b6b40cb47b6a1f5876106eaa8d2 2024-11-09T06:18:23,621 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09667b6b40cb47b6a1f5876106eaa8d2, entries=8, sequenceid=2155, filesize=5.5 K 2024-11-09T06:18:23,622 DEBUG [M:0;4a84bcce7bad:37069 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4331b7c4bb0e42aa990b2485b82c8080 as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4331b7c4bb0e42aa990b2485b82c8080 2024-11-09T06:18:23,625 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4331b7c4bb0e42aa990b2485b82c8080 2024-11-09T06:18:23,625 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4331b7c4bb0e42aa990b2485b82c8080, entries=164, sequenceid=2155, filesize=43.3 K 2024-11-09T06:18:23,625 DEBUG [M:0;4a84bcce7bad:37069 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d6053347b6ea42779199f5dbfcf176bb as hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d6053347b6ea42779199f5dbfcf176bb 2024-11-09T06:18:23,628 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43365/user/jenkins/test-data/0c8bc657-ac3b-058d-362a-fcf2bf272e3f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d6053347b6ea42779199f5dbfcf176bb, entries=1, sequenceid=2155, filesize=5.0 K 2024-11-09T06:18:23,629 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.HRegion(3140): Finished flush of dataSize ~774.35 KB/792937, heapSize ~948.30 KB/971056, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 899ms, sequenceid=2155, compaction requested=false 2024-11-09T06:18:23,634 INFO [M:0;4a84bcce7bad:37069 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T06:18:23,634 DEBUG [M:0;4a84bcce7bad:37069 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731133102730Disabling compacts and flushes for region at 1731133102730Disabling writes for close at 1731133102730Obtaining lock to block concurrent updates at 1731133102730Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731133102730Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=792937, getHeapSize=971296, getOffHeapSize=0, getCellsCount=2344 at 1731133102731 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731133102731Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731133102731Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731133102749 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731133102749Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731133103157 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731133103173 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731133103173Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731133103593 (+420 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731133103610 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731133103610Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c6dca17: reopening flushed file at 1731133103617 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@167d39: reopening flushed file at 1731133103621 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e78bbf4: reopening flushed file at 1731133103625 (+4 ms)Finished flush of dataSize ~774.35 KB/792937, heapSize ~948.30 KB/971056, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 899ms, sequenceid=2155, compaction requested=false at 1731133103629 (+4 ms)Writing region close event to WAL at 1731133103634 (+5 ms)Closed at 1731133103634 2024-11-09T06:18:23,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41571 is added to blk_1073741830_1006 (size=934187) 2024-11-09T06:18:23,638 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T06:18:23,639 INFO [M:0;4a84bcce7bad:37069 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T06:18:23,639 INFO [M:0;4a84bcce7bad:37069 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37069 2024-11-09T06:18:23,639 INFO [M:0;4a84bcce7bad:37069 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T06:18:23,797 INFO [M:0;4a84bcce7bad:37069 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T06:18:23,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T06:18:23,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37069-0x1011e3c29400000, quorum=127.0.0.1:62248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T06:18:23,801 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T06:18:23,803 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T06:18:23,804 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T06:18:23,804 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T06:18:23,804 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/hadoop.log.dir/,STOPPED} 2024-11-09T06:18:23,807 WARN [BP-1465325163-172.17.0.3-1731132923455 heartbeating to localhost/127.0.0.1:43365 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T06:18:23,807 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T06:18:23,807 WARN [BP-1465325163-172.17.0.3-1731132923455 heartbeating to localhost/127.0.0.1:43365 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1465325163-172.17.0.3-1731132923455 (Datanode Uuid fa7624cf-f0f4-47e0-ac1f-fe9b1ca50877) service to localhost/127.0.0.1:43365 2024-11-09T06:18:23,807 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T06:18:23,810 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/cluster_d5b7a4cc-a1e2-6520-f0a5-179e578abc6c/data/data1/current/BP-1465325163-172.17.0.3-1731132923455 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T06:18:23,811 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/cluster_d5b7a4cc-a1e2-6520-f0a5-179e578abc6c/data/data2/current/BP-1465325163-172.17.0.3-1731132923455 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T06:18:23,812 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T06:18:23,820 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T06:18:23,821 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T06:18:23,821 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T06:18:23,821 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T06:18:23,821 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/13199b32-4299-6d38-ccbf-81fc5807dfe8/hadoop.log.dir/,STOPPED} 2024-11-09T06:18:23,837 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T06:18:24,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down